diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-18 15:09:42 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-18 15:52:30 +0000 |
commit | b2fc6977048321196ff320f6af744616d73a0615 (patch) | |
tree | b8523cf5b7b5045772df656a9157897abcdc5f15 | |
parent | 24f8d5ab63a193aeb6660eafb83a22956a679a75 (diff) |
Report peak memory usage.
-rw-r--r-- | eval/src/apps/analyze_onnx_model/CMakeLists.txt | 2 | ||||
-rw-r--r-- | eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp | 17 | ||||
-rw-r--r-- | vespamalloc/src/tests/stacktrace/stacktrace.cpp | 4 | ||||
-rw-r--r-- | vespamalloc/src/vespamalloc/malloc/mmappool.cpp | 12 | ||||
-rw-r--r-- | vespamalloc/src/vespamalloc/malloc/mmappool.h | 2 | ||||
-rw-r--r-- | vespamalloc/src/vespamalloc/malloc/overload.h | 8 |
6 files changed, 29 insertions, 16 deletions
diff --git a/eval/src/apps/analyze_onnx_model/CMakeLists.txt b/eval/src/apps/analyze_onnx_model/CMakeLists.txt index a8f984550f9..6ab7a17e109 100644 --- a/eval/src/apps/analyze_onnx_model/CMakeLists.txt +++ b/eval/src/apps/analyze_onnx_model/CMakeLists.txt @@ -6,4 +6,6 @@ vespa_add_executable(eval_analyze_onnx_model_app INSTALL bin DEPENDS vespaeval + EXTERNAL_DEPENDS + dl ) diff --git a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp index 2358a6c263e..1fdc6fe79f0 100644 --- a/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp +++ b/eval/src/apps/analyze_onnx_model/analyze_onnx_model.cpp @@ -10,6 +10,8 @@ #include <vespa/vespalib/util/guard.h> #include <vespa/vespalib/util/stringfmt.h> #include <charconv> +#include <malloc.h> +#include <dlfcn.h> using vespalib::make_string_short::fmt; @@ -75,17 +77,12 @@ size_t convert(const vespalib::string & s) { } MemoryUsage extract_memory_usage() { - vespalib::string vm_size = UNKNOWN; - vespalib::string vm_rss = UNKNOWN; - FilePointer file(fopen("/proc/self/status", "r")); - if (file.valid()) { - vespalib::string line; - while (read_line(file, line)) { - extract(line, "VmSize:", vm_size); - extract(line, "VmRSS:", vm_rss); - } + struct mallinfo info = mallinfo(); + if (dlsym(RTLD_NEXT, "is_vespamalloc") != nullptr) { + return {size_t(info.usmblks) << 20, size_t(info.arena + info.hblkhd) << 20}; + } else { + return {size_t(info.usmblks), size_t(info.arena + info.hblkhd)}; } - return {convert(vm_size), convert(vm_rss)}; } void report_memory_usage(const vespalib::string &desc) { diff --git a/vespamalloc/src/tests/stacktrace/stacktrace.cpp b/vespamalloc/src/tests/stacktrace/stacktrace.cpp index d7f7d45656d..2592222fce3 100644 --- a/vespamalloc/src/tests/stacktrace/stacktrace.cpp +++ b/vespamalloc/src/tests/stacktrace/stacktrace.cpp @@ -29,7 +29,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.ordblks == 0); assert(info.smblks == 0); assert(info.uordblks > 0); - assert(info.usmblks == 0); + assert(info.usmblks > 0); #else struct mallinfo info = mallinfo(); printf("Malloc used %dm of memory\n",info.arena); @@ -43,7 +43,7 @@ void verify_that_vespamalloc_datasegment_size_exists() { assert(info.ordblks == 0); assert(info.smblks == 0); assert(info.uordblks > 0); - assert(info.usmblks == 0); + assert(info.usmblks > 0); #endif } diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp index e43ea2e8342..ca728547548 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.cpp +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.cpp @@ -10,6 +10,7 @@ namespace vespamalloc { MMapPool::MMapPool() : _page_size(getpagesize()), _huge_flags((getenv("VESPA_USE_HUGEPAGES") != nullptr) ? MAP_HUGETLB : 0), + _peakBytes(0ul), _count(0), _mutex(), _mappings() @@ -33,6 +34,12 @@ MMapPool::getMmappedBytes() const { return sum; } +size_t +MMapPool::getMmappedBytesPeak() const { + std::lock_guard guard(_mutex); + return _peakBytes; +} + void * MMapPool::mmap(size_t sz) { void * buf(nullptr); @@ -76,9 +83,10 @@ MMapPool::mmap(size_t sz) { std::lock_guard guard(_mutex); auto [it, inserted] = _mappings.insert(std::make_pair(buf, MMapInfo(mmapId, sz))); ASSERT_STACKTRACE(inserted); + size_t sum(0); + std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); + _peakBytes = std::max(_peakBytes, sum); if (sz >= _G_bigBlockLimit) { - size_t sum(0); - std::for_each(_mappings.begin(), _mappings.end(), [&sum](const auto & e){ sum += e.second._sz; }); fprintf(_G_logFile, "%ld mappings of accumulated size %ld\n", _mappings.size(), sum); } } diff --git a/vespamalloc/src/vespamalloc/malloc/mmappool.h b/vespamalloc/src/vespamalloc/malloc/mmappool.h index c0b73c56e4e..fe02b33f82f 100644 --- a/vespamalloc/src/vespamalloc/malloc/mmappool.h +++ b/vespamalloc/src/vespamalloc/malloc/mmappool.h @@ -19,6 +19,7 @@ public: size_t get_size(void *) const; size_t getNumMappings() const; size_t getMmappedBytes() const; + size_t getMmappedBytesPeak() const; void info(FILE * os, size_t level) const; private: struct MMapInfo { @@ -28,6 +29,7 @@ private: }; const size_t _page_size; const int _huge_flags; + size_t _peakBytes; std::atomic<size_t> _count; std::atomic<bool> _has_hugepage_failure_just_happened; mutable std::mutex _mutex; diff --git a/vespamalloc/src/vespamalloc/malloc/overload.h b/vespamalloc/src/vespamalloc/malloc/overload.h index cb57e6b2fa8..bf9710b9b32 100644 --- a/vespamalloc/src/vespamalloc/malloc/overload.h +++ b/vespamalloc/src/vespamalloc/malloc/overload.h @@ -119,7 +119,9 @@ struct mallinfo2 mallinfo2() __THROW { info.smblks = 0; info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); info.hblks = vespamalloc::_GmemP->mmapPool().getMmappedBytes(); - info.usmblks = 0; + size_t highwaterMark = vespamalloc::_GmemP->dataSegment().dataSize() + + vespamalloc::_GmemP->mmapPool().getMmappedBytesPeak(); + info.usmblks = highwaterMark; info.fsmblks = 0; info.fordblks = vespamalloc::_GmemP->dataSegment().freeSize(); info.uordblks = info.arena + info.hblks - info.fordblks; @@ -135,7 +137,9 @@ struct mallinfo mallinfo() __THROW { info.smblks = 0; info.hblkhd = vespamalloc::_GmemP->mmapPool().getNumMappings(); info.hblks = (vespamalloc::_GmemP->mmapPool().getMmappedBytes() >> 20); - info.usmblks = 0; + size_t highwaterMark = vespamalloc::_GmemP->dataSegment().dataSize() + + vespamalloc::_GmemP->mmapPool().getMmappedBytesPeak(); + info.usmblks = (highwaterMark >> 20); info.fsmblks = 0; info.fordblks = (vespamalloc::_GmemP->dataSegment().freeSize() >> 20); info.uordblks = info.arena + info.hblks - info.fordblks; |