Ensure consistent memory stats snapshot by a process.

3 files changed, 99 insertions, 35 deletions

diff --git a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp
index c8a9590eb96..6210acb1f1a 100644
--- a/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp
+++ b/staging_vespalib/src/tests/util/process_memory_stats/process_memory_stats_test.cpp
@@ -27,7 +27,7 @@ std::string toString(const ProcessMemoryStats &stats)
 
 TEST("Simple stats")
 {
-    ProcessMemoryStats stats(ProcessMemoryStats::create());
+    ProcessMemoryStats stats(ProcessMemoryStats::create(4096));
     std::cout << toString(stats) << std::endl;
     EXPECT_LESS(0u, stats.getMappedVirt());
     EXPECT_LESS(0u, stats.getMappedRss());
@@ -37,7 +37,7 @@ TEST("Simple stats")
 
 TEST("grow anonymous memory")
 {
-    ProcessMemoryStats stats1(ProcessMemoryStats::create());
+    ProcessMemoryStats stats1(ProcessMemoryStats::create(4096));
     std::cout << toString(stats1) << std::endl;
     size_t mapLen = 64 * 1024;
     void *mapAddr = mmap(nullptr, mapLen, PROT_READ | PROT_WRITE,
@@ -48,7 +48,7 @@ TEST("grow anonymous memory")
     EXPECT_LESS_EQUAL(stats1.getAnonymousVirt() + mapLen,
                       stats2.getAnonymousVirt());
     memset(mapAddr, 1, mapLen);
-    ProcessMemoryStats stats3(ProcessMemoryStats::create());
+    ProcessMemoryStats stats3(ProcessMemoryStats::create(4096));
     std::cout << toString(stats3) << std::endl;
     // Cannot check that resident grows if swap is enabled and system loaded
     munmap(mapAddr, mapLen);
@@ -63,16 +63,16 @@ TEST("grow mapped memory")
     of.close();
     int mapfileFileDescriptor = open("mapfile", O_RDONLY, 0666);
     EXPECT_LESS_EQUAL(0, mapfileFileDescriptor);
-    ProcessMemoryStats stats1(ProcessMemoryStats::create());
+    ProcessMemoryStats stats1(ProcessMemoryStats::create(4096));
     std::cout << toString(stats1) << std::endl;
     void *mapAddr = mmap(nullptr, mapLen, PROT_READ, MAP_SHARED,
                          mapfileFileDescriptor, 0);
     EXPECT_NOT_EQUAL(reinterpret_cast<void *>(-1), mapAddr);
-    ProcessMemoryStats stats2(ProcessMemoryStats::create());
+    ProcessMemoryStats stats2(ProcessMemoryStats::create(4096));
     std::cout << toString(stats2) << std::endl;
     EXPECT_LESS_EQUAL(stats1.getMappedVirt() + mapLen, stats2.getMappedVirt());
     EXPECT_EQUAL(0, memcmp(mapAddr, &buf[0], mapLen));
-    ProcessMemoryStats stats3(ProcessMemoryStats::create());
+    ProcessMemoryStats stats3(ProcessMemoryStats::create(4096));
     std::cout << toString(stats3) << std::endl;
     // Cannot check that resident grows if swap is enabled and system loaded
     munmap(mapAddr, mapLen);
diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp
index 849ebf6d4fa..13e2b3085ca 100644
--- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp
+++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.cpp
@@ -1,11 +1,15 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "process_memory_stats.h"
+#include <vespa/vespalib/stllike/asciistream.h>
 #include <fstream>
 #include <sstream>
 
-namespace vespalib
-{
+#include <vespa/log/log.h>
+
+LOG_SETUP(".vespalib.util.process_memory_stats");
+
+namespace vespalib {
 
 namespace {
 
@@ -84,31 +88,8 @@ std::string getLineHeader(const std::string &line)
 
 }
 
-
-ProcessMemoryStats::ProcessMemoryStats()
-    : _mapped_virt(0),
-      _mapped_rss(0),
-      _anonymous_virt(0),
-      _anonymous_rss(0),
-      _mappings_count(0)
-{
-}
-
-ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt,
-                                       uint64_t mapped_rss,
-                                       uint64_t anonymous_virt,
-                                       uint64_t anonymous_rss,
-                                       uint64_t mappings_cnt)
-    : _mapped_virt(mapped_virt),
-      _mapped_rss(mapped_rss),
-      _anonymous_virt(anonymous_virt),
-      _anonymous_rss(anonymous_rss),
-      _mappings_count(mappings_cnt)
-{
-}
-
 ProcessMemoryStats
-ProcessMemoryStats::create()
+ProcessMemoryStats::createStatsFromSmaps()
 {
     ProcessMemoryStats ret;
     std::ifstream smaps("/proc/self/smaps");
@@ -143,4 +124,76 @@ ProcessMemoryStats::create()
     return ret;
 }
 
-} // namespace vespalib
+
+ProcessMemoryStats::ProcessMemoryStats()
+    : _mapped_virt(0),
+      _mapped_rss(0),
+      _anonymous_virt(0),
+      _anonymous_rss(0),
+      _mappings_count(0)
+{
+}
+
+ProcessMemoryStats::ProcessMemoryStats(uint64_t mapped_virt,
+                                       uint64_t mapped_rss,
+                                       uint64_t anonymous_virt,
+                                       uint64_t anonymous_rss,
+                                       uint64_t mappings_cnt)
+    : _mapped_virt(mapped_virt),
+      _mapped_rss(mapped_rss),
+      _anonymous_virt(anonymous_virt),
+      _anonymous_rss(anonymous_rss),
+      _mappings_count(mappings_cnt)
+{
+}
+
+namespace {
+
+bool
+similar(uint64_t lhs, uint64_t rhs, uint64_t epsilon)
+{
+    return (lhs < rhs) ? ((rhs - lhs) < epsilon) : ((lhs - rhs) < epsilon);
+}
+
+}
+
+bool
+ProcessMemoryStats::similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const
+{
+    return similar(_mapped_virt, rhs._mapped_virt, sizeEpsilon) &&
+            similar(_mapped_rss, rhs._mapped_rss, sizeEpsilon) &&
+            similar(_anonymous_virt, rhs._anonymous_virt, sizeEpsilon) &&
+            similar(_anonymous_rss, rhs._anonymous_rss, sizeEpsilon) &&
+            (_mappings_count == rhs._mappings_count);
+}
+
+vespalib::string
+ProcessMemoryStats::toString() const
+{
+    vespalib::asciistream stream;
+    stream << "_mapped_virt=" << _mapped_virt << ", "
+           << "_mapped_rss=" << _mapped_rss << ", "
+           << "_anonymous_virt=" << _anonymous_virt << ", "
+           << "_anonymous_rss=" << _anonymous_rss << ", "
+           << "_mappings_count=" << _mappings_count;
+    return stream.str();
+}
+
+ProcessMemoryStats
+ProcessMemoryStats::create(uint64_t sizeEpsilon)
+{
+    ProcessMemoryStats prevStats = createStatsFromSmaps();
+    const size_t NUM_TRIES = 10;
+    for (size_t i = 0; i < NUM_TRIES; ++i) {
+        ProcessMemoryStats currStats = createStatsFromSmaps();
+        if (prevStats.similarTo(currStats, sizeEpsilon)) {
+            return prevStats;
+        }
+        LOG(info, "create(): Memory stats have changed, trying to read smaps file again: i=%zu, prevStats={%s}, currStats={%s}",
+            i, prevStats.toString().c_str(), currStats.toString().c_str());
+        prevStats = currStats;
+    }
+    return prevStats;
+}
+
+}
diff --git a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h
index b864a509eaa..fe5062f75cd 100644
--- a/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h
+++ b/staging_vespalib/src/vespa/vespalib/util/process_memory_stats.h
@@ -3,6 +3,7 @@
 #pragma once
 
 #include <cstdint>
+#include <vespa/vespalib/stllike/string.h>
 
 namespace vespalib {
 
@@ -18,17 +19,27 @@ class ProcessMemoryStats
     uint64_t _mappings_count; // number of mappings
                               // (limited by sysctl vm.max_map_count)
 
+    static ProcessMemoryStats createStatsFromSmaps();
+
 public:
     ProcessMemoryStats();
-    static ProcessMemoryStats create(); // based on /proc/self/smaps
+    /**
+     * Sample memory stats for the current process based on reading the file /proc/self/smaps.
+     *
+     * Samples are taken until two consecutive memory stats are similar given the size epsilon.
+     * This ensures a somewhat consistent memory stats snapshot.
+     */
+    static ProcessMemoryStats create(uint64_t sizeEpsilon = 1 * 1024 * 1024);
     uint64_t getMappedVirt() const { return _mapped_virt; }
     uint64_t getMappedRss() const { return _mapped_rss; }
     uint64_t getAnonymousVirt() const { return _anonymous_virt; }
     uint64_t getAnonymousRss() const { return _anonymous_rss; }
     uint64_t getMappingsCount() const { return _mappings_count; }
+    bool similarTo(const ProcessMemoryStats &rhs, uint64_t sizeEpsilon) const;
+    vespalib::string toString() const;
 
     /** for unit tests only */
     ProcessMemoryStats(uint64_t, uint64_t, uint64_t, uint64_t, uint64_t);
 };
 
-} // namespace vespalib
+}