diff options
author | Geir Storli <geirst@oath.com> | 2018-05-08 15:18:24 +0000 |
---|---|---|
committer | Geir Storli <geirst@oath.com> | 2018-05-08 15:18:24 +0000 |
commit | ffcdd640393978666c08e40ed02764d2fc3e8c42 (patch) | |
tree | ad7be6f0832b964d94697927d97455ae9e1c5c63 /staging_vespalib/src | |
parent | d7911658858efddf1478018968525d4cc4fa8d6e (diff) |
Ensure consistent memory stats snapshot by a process.
Diffstat (limited to 'staging_vespalib/src')
3 files changed, 99 insertions, 35 deletions
diff --git a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp index c8a9590eb96..6210acb1f1a 100644 --- a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp +++ b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp @@ -27,7 +27,7 @@ std::string toString(const ProcessMemoryStats &stats) TEST("Simple stats") { - ProcessMemoryStats stats(ProcessMemoryStats::create()); + ProcessMemoryStats stats(ProcessMemoryStats::create(4096)); std::cout << toString(stats) << std::endl; EXPECT_LESS(0u, stats.getMappedVirt()); EXPECT_LESS(0u, stats.getMappedRss()); @@ -37,7 +37,7 @@ TEST("Simple stats") TEST("grow anonymous memory") { - ProcessMemoryStats stats1(ProcessMemoryStats::create()); + ProcessMemoryStats stats1(ProcessMemoryStats::create(4096)); std::cout << toString(stats1) << std::endl; size_t mapLen = 64 * 1024; void *mapAddr = mmap(nullptr, mapLen, PROT_READ | PROT_WRITE, @@ -48,7 +48,7 @@ TEST("grow anonymous memory") EXPECT_LESS_EQUAL(stats1.getAnonymousVirt() + mapLen, stats2.getAnonymousVirt()); memset(mapAddr, 1, mapLen); - ProcessMemoryStats stats3(ProcessMemoryStats::create()); + ProcessMemoryStats stats3(ProcessMemoryStats::create(4096)); std::cout << toString(stats3) << std::endl; // Cannot check that resident grows if swap is enabled and system loaded munmap(mapAddr, mapLen); @@ -63,16 +63,16 @@ TEST("grow mapped memory") of.close(); int mapfileFileDescriptor = open("mapfile", O_RDONLY, 0666); EXPECT_LESS_EQUAL(0, mapfileFileDescriptor); - ProcessMemoryStats stats1(ProcessMemoryStats::create()); + ProcessMemoryStats stats1(ProcessMemoryStats::create(4096)); std::cout << toString(stats1) << std::endl; void *mapAddr = mmap(nullptr, mapLen, PROT_READ, MAP_SHARED, mapfileFileDescriptor, 0); EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr); - ProcessMemoryStats stats2(ProcessMemoryStats::create()); + ProcessMemoryStats stats2(ProcessMemoryStats::create(4096)); std::cout << toString(stats2) << std::endl; EXPECT_LESS_EQUAL(stats1.getMappedVirt() + mapLen, stats2.getMappedVirt()); EXPECT_EQUAL(0, memcmp(mapAddr, &buf[0], mapLen)); - ProcessMemoryStats stats3(ProcessMemoryStats::create()); + ProcessMemoryStats stats3(ProcessMemoryStats::create(4096)); std::cout << toString(stats3) << std::endl; // Cannot check that resident grows if swap is enabled and system loaded munmap(mapAddr, mapLen); diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp index 849ebf6d4fa..13e2b3085ca 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp @@ -1,11 +1,15 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "process_memory_stats.h" +#include <vespa/vespalib/stllike/asciistream.h> #include <fstream> #include <sstream> -namespace vespalib -{ +#include <vespa/log/log.h> + +LOG_SETUP(".vespalib.util.process_memory_stats"); + +namespace vespalib { namespace { @@ -84,31 +88,8 @@ std::string getLineHeader(const std::string &line) } - -ProcessMemoryStats::ProcessMemoryStats() - : _mapped_virt(0), - _mapped_rss(0), - _anonymous_virt(0), - _anonymous_rss(0), - _mappings_count(0) -{ -} - -ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt, - uint64_t mapped_rss, - uint64_t anonymous_virt, - uint64_t anonymous_rss, - uint64_t mappings_cnt) - : _mapped_virt(mapped_virt), - _mapped_rss(mapped_rss), - _anonymous_virt(anonymous_virt), - _anonymous_rss(anonymous_rss), - _mappings_count(mappings_cnt) -{ -} - ProcessMemoryStats -ProcessMemoryStats::create() +ProcessMemoryStats::createStatsFromSmaps() { ProcessMemoryStats ret; std::ifstream smaps("/proc/self/smaps"); @@ -143,4 +124,76 @@ ProcessMemoryStats::create() return ret; } -} // namespace vespalib + +ProcessMemoryStats::ProcessMemoryStats() + : _mapped_virt(0), + _mapped_rss(0), + _anonymous_virt(0), + _anonymous_rss(0), + _mappings_count(0) +{ +} + +ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt, + uint64_t mapped_rss, + uint64_t anonymous_virt, + uint64_t anonymous_rss, + uint64_t mappings_cnt) + : _mapped_virt(mapped_virt), + _mapped_rss(mapped_rss), + _anonymous_virt(anonymous_virt), + _anonymous_rss(anonymous_rss), + _mappings_count(mappings_cnt) +{ +} + +namespace { + +bool +similar(uint64_t lhs, uint64_t rhs, uint64_t epsilon) +{ + return (lhs < rhs) ? ((rhs - lhs) < epsilon) : ((lhs - rhs) < epsilon); +} + +} + +bool +ProcessMemoryStats::similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const +{ + return similar(_mapped_virt, rhs._mapped_virt, sizeEpsilon) && + similar(_mapped_rss, rhs._mapped_rss, sizeEpsilon) && + similar(_anonymous_virt, rhs._anonymous_virt, sizeEpsilon) && + similar(_anonymous_rss, rhs._anonymous_rss, sizeEpsilon) && + (_mappings_count == rhs._mappings_count); +} + +vespalib::string +ProcessMemoryStats::toString() const +{ + vespalib::asciistream stream; + stream << "_mapped_virt=" << _mapped_virt << ", " + << "_mapped_rss=" << _mapped_rss << ", " + << "_anonymous_virt=" << _anonymous_virt << ", " + << "_anonymous_rss=" << _anonymous_rss << ", " + << "_mappings_count=" << _mappings_count; + return stream.str(); +} + +ProcessMemoryStats +ProcessMemoryStats::create(uint64_t sizeEpsilon) +{ + ProcessMemoryStats prevStats = createStatsFromSmaps(); + const size_t NUM_TRIES = 10; + for (size_t i = 0; i < NUM_TRIES; ++i) { + ProcessMemoryStats currStats = createStatsFromSmaps(); + if (prevStats.similarTo(currStats, sizeEpsilon)) { + return prevStats; + } + LOG(info, "create(): Memory stats have changed, trying to read smaps file again: i=%zu, prevStats={%s}, currStats={%s}", + i, prevStats.toString().c_str(), currStats.toString().c_str()); + prevStats = currStats; + } + return prevStats; +} + +} diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h index b864a509eaa..fe5062f75cd 100644 --- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h +++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h @@ -3,6 +3,7 @@ #pragma once #include <cstdint> +#include <vespa/vespalib/stllike/string.h> namespace vespalib { @@ -18,17 +19,27 @@ class ProcessMemoryStats uint64_t _mappings_count; // number of mappings // (limited by sysctl vm.max_map_count) + static ProcessMemoryStats createStatsFromSmaps(); + public: ProcessMemoryStats(); - static ProcessMemoryStats create(); // based on /proc/self/smaps + /** + * Sample memory stats for the current process based on reading the file /proc/self/smaps. + * + * Samples are taken until two consecutive memory stats are similar given the size epsilon. + * This ensures a somewhat consistent memory stats snapshot. + */ + static ProcessMemoryStats create(uint64_t sizeEpsilon = 1 * 1024 * 1024); uint64_t getMappedVirt() const { return _mapped_virt; } uint64_t getMappedRss() const { return _mapped_rss; } uint64_t getAnonymousVirt() const { return _anonymous_virt; } uint64_t getAnonymousRss() const { return _anonymous_rss; } uint64_t getMappingsCount() const { return _mappings_count; } + bool similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const; + vespalib::string toString() const; /** for unit tests only */ ProcessMemoryStats(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t); }; -} // namespace vespalib +} |