Change DiskMemUsageSampler to sample disk usage using directory scan if disk is shared, otherwise looking at file system info.

This should fix sampling of disk usage when using docker.

7 files changed, 107 insertions, 79 deletions

diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
index 5d8a647f675..cfd1c37e4d1 100644
--- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
@@ -1,9 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h>
 
 using proton::DiskMemUsageFilter;
+using proton::HwInfo;
 
 namespace fs = std::experimental::filesystem;
 
@@ -16,9 +18,9 @@ struct Fixture
     using Config = DiskMemUsageFilter::Config;
 
     Fixture()
-        : _filter(64 * 1024 * 1024)
+        : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(64 * 1024 * 1024)))
     {
-        _filter.setDiskStats({.capacity = 100, .free = 100, .available=100});
+        _filter.setDiskUsedSize(0);
         _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000,
                                                             10000001,
                                                             10000002,
@@ -41,7 +43,7 @@ struct Fixture
     }
 
     void triggerDiskLimit() {
-        _filter.setDiskStats({.capacity = 100, .free = 20, .available=10});
+        _filter.setDiskUsedSize(90);
     }
 
     void triggerMemoryLimit()
@@ -76,7 +78,7 @@ TEST_F("Check that disk limit can be reached", Fixture)
                 "action: \"add more content nodes\", "
                 "reason: \"disk used (0.9) > disk limit (0.8)\", "
                 "stats: { "
-                "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}");
+                "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}");
 }
 
 TEST_F("Check that memory limit can be reached", Fixture)
@@ -108,7 +110,7 @@ TEST_F("Check that both disk limit and memory limit can be reached", Fixture)
                 "action: \"add more content nodes\", "
                 "reason: \"disk used (0.9) > disk limit (0.8)\", "
                 "stats: { "
-                "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}");
+                "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}");
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index d489f477df2..32b2b6aaba6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -3,6 +3,7 @@
 #include "disk_mem_usage_filter.h"
 #include "i_disk_mem_usage_listener.h"
 #include <vespa/log/log.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 
 LOG_SETUP(".proton.server.disk_mem_usage_filter");
 
@@ -43,12 +44,12 @@ void
 makeDiskStatsMessage(std::ostream &os,
                      double diskUsed,
                      double diskLimit,
-                     const DiskMemUsageFilter::space_info &diskStats)
+                     const HwInfo &hwInfo,
+                     uint64_t usedDiskSizeBytes)
 {
     os << "stats: { ";
-    os << "capacity: " << diskStats.capacity << ", ";
-    os << "free: " << diskStats.free << ", ";
-    os << "available: " << diskStats.available << ", ";
+    os << "capacity: " << hwInfo.disk().sizeBytes() << ", ";
+    os << "used: " << usedDiskSizeBytes << ", ";
     os << "diskUsed: " << diskUsed << ", ";
     os << "diskLimit: " << diskLimit << "}";
 }
@@ -57,31 +58,31 @@ void
 makeDiskLimitMessage(std::ostream &os,
                      double diskUsed,
                      double diskLimit,
-                     const DiskMemUsageFilter::space_info &diskStats)
+                     const HwInfo &hwInfo,
+                     uint64_t usedDiskSizeBytes)
 {
     os << "diskLimitReached: { ";
     os << "action: \"add more content nodes\", ";
     os << "reason: \"disk used (" << diskUsed << ") > disk limit (" << diskLimit << ")\", ";
-    makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats);
+    makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes);
     os << "}";
 }
 
-
 vespalib::string
 makeUnblockingMessage(double memoryUsed,
                       double memoryLimit,
                       const vespalib::ProcessMemoryStats &memoryStats,
-                      uint64_t physicalMemory,
+                      const HwInfo &hwInfo,
                       double diskUsed,
                       double diskLimit,
-                      const DiskMemUsageFilter::space_info &diskStats)
+                      uint64_t usedDiskSizeBytes)
 {
     std::ostringstream os;
     os << "memoryLimitOK: { ";
-    makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, physicalMemory);
+    makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, hwInfo.memory().sizeBytes());
     os << "}, ";
     os << "diskLimitOK: { ";
-    makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats);
+    makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes);
     os << "}";
     return os.str();
 }
@@ -97,7 +98,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
     if (memoryUsed > _config._memoryLimit) {
         hasMessage = true;
         makeMemoryLimitMessage(message, memoryUsed,
-                _config._memoryLimit, _memoryStats, _physicalMemory);
+                _config._memoryLimit, _memoryStats, _hwInfo.memory().sizeBytes());
     }
     double diskUsed = getDiskUsedRatio(guard);
     if (diskUsed > _config._diskLimit) {
@@ -105,7 +106,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
             message << ", ";
         }
         hasMessage = true;
-        makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _diskStats);
+        makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _hwInfo, _diskUsedSizeBytes);
     }
     if (hasMessage) {
         if (_acceptWrite) {
@@ -118,10 +119,10 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
             vespalib::string unblockMsg = makeUnblockingMessage(memoryUsed,
                                                                 _config._memoryLimit,
                                                                 _memoryStats,
-                                                                _physicalMemory,
+                                                                _hwInfo,
                                                                 diskUsed,
                                                                 _config._diskLimit,
-                                                                _diskStats);
+                                                                _diskUsedSizeBytes);
             LOG(info, "Write operations are now un-blocked: '%s'", unblockMsg.c_str());
         }
         _state = State();
@@ -137,23 +138,23 @@ DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const
 {
     (void) guard;
     uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss();
-    return static_cast<double>(unscaledMemoryUsed) / _physicalMemory;
+    return static_cast<double>(unscaledMemoryUsed) / _hwInfo.memory().sizeBytes();
 }
 
 double
 DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const
 {
     (void) guard;
-    double availableDiskSpaceRatio = static_cast<double>(_diskStats.available) /
-                                     static_cast<double>(_diskStats.capacity);
-    return 1.0 - availableDiskSpaceRatio;
+    double usedDiskSpaceRatio = static_cast<double>(_diskUsedSizeBytes) /
+                                static_cast<double>(_hwInfo.disk().sizeBytes());
+    return usedDiskSpaceRatio;
 }
 
-DiskMemUsageFilter::DiskMemUsageFilter(uint64_t physicalMemory_in)
+DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
     : _lock(),
+      _hwInfo(hwInfo),
       _memoryStats(),
-      _physicalMemory(physicalMemory_in),
-      _diskStats(),
+      _diskUsedSizeBytes(),
       _config(),
       _state(),
       _acceptWrite(true),
@@ -172,10 +173,10 @@ DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in)
 }
 
 void
-DiskMemUsageFilter::setDiskStats(space_info diskStats_in)
+DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes)
 {
     Guard guard(_lock);
-    _diskStats = diskStats_in;
+    _diskUsedSizeBytes = diskUsedSizeBytes;
     recalcState(guard);
 }
 
@@ -194,11 +195,11 @@ DiskMemUsageFilter::getMemoryStats() const
     return _memoryStats;
 }
 
-DiskMemUsageFilter::space_info
-DiskMemUsageFilter::getDiskStats() const
+uint64_t
+DiskMemUsageFilter::getDiskUsedSize() const
 {
     Guard guard(_lock);
-    return _diskStats;
+    return _diskUsedSizeBytes;
 }
 
 DiskMemUsageFilter::Config
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
index c06d2e2ab0a..4906348a54d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -4,12 +4,12 @@
 
 #include "i_disk_mem_usage_notifier.h"
 #include "disk_mem_usage_state.h"
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
 #include <vespa/vespalib/util/process_memory_stats.h>
-#include <mutex>
 #include <atomic>
 #include <experimental/filesystem>
-
+#include <mutex>
 
 namespace proton {
 
@@ -39,10 +39,10 @@ public:
     };
 
 private:
-    mutable Mutex _lock; // protect _memoryStats, _diskStats, _config, _state
+    mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state
+    HwInfo                       _hwInfo;
     vespalib::ProcessMemoryStats _memoryStats;
-    uint64_t                     _physicalMemory;
-    space_info                   _diskStats;
+    uint64_t                     _diskUsedSizeBytes;
     Config                       _config;
     State                        _state;
     std::atomic<bool>            _acceptWrite;
@@ -55,15 +55,15 @@ private:
     void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state);
 
 public:
-    DiskMemUsageFilter(uint64_t physicalMememory_in);
+    DiskMemUsageFilter(const HwInfo &hwInfo);
     ~DiskMemUsageFilter();
     void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in);
-    void setDiskStats(space_info diskStats_in);
+    void setDiskUsedSize(uint64_t diskUsedSizeBytes);
     void setConfig(Config config);
     vespalib::ProcessMemoryStats getMemoryStats() const;
-    space_info getDiskStats() const;
+    uint64_t getDiskUsedSize() const;
     Config getConfig() const;
-    uint64_t getPhysicalMemory() const { return _physicalMemory; }
+    const HwInfo &getHwInfo() const { return _hwInfo; }
     double getMemoryUsedRatio() const;
     double getDiskUsedRatio() const;
     bool acceptWriteOperation() const override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
index bacf80e69a6..ddbede13880 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
@@ -3,30 +3,16 @@
 #include "disk_mem_usage_sampler.h"
 #include <vespa/vespalib/util/timer.h>
 #include <vespa/vespalib/util/lambdatask.h>
+#include <experimental/filesystem>
 #include <unistd.h>
 
 using vespalib::makeLambdaTask;
 
 namespace proton {
 
-namespace {
-
-uint64_t getPhysicalMemoryBytes()
-{
-    // TODO: Temporal workaround for Docker nodes. Remove when this is part of proton.cfg instead.
-    if (const char *memoryEnv = std::getenv("VESPA_TOTAL_MEMORY_MB")) {
-        uint64_t physicalMemoryMB = atoll(memoryEnv);
-        return physicalMemoryMB * 1024u * 1024u;
-    } else {
-        return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
-    }
-}
-
-} // namespace proton:<anonymous>
-
 DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in,
                                          const Config &config)
-    : _filter(getPhysicalMemoryBytes()),
+    : _filter(config.hwInfo),
       _path(path_in),
       _sampleInterval(60.0),
       _periodicTimer()
@@ -43,8 +29,8 @@ void
 DiskMemUsageSampler::setConfig(const Config &config)
 {
     _periodicTimer.reset();
-    _filter.setConfig(config._filterConfig);
-    _sampleInterval = config._sampleInterval;
+    _filter.setConfig(config.filterConfig);
+    _sampleInterval = config.sampleInterval;
     sampleUsage();
     _periodicTimer = std::make_unique<vespalib::Timer>();
     _periodicTimer->scheduleAtFixedRate(makeLambdaTask([this]()
@@ -59,10 +45,43 @@ DiskMemUsageSampler::sampleUsage()
     sampleDiskUsage();
 }
 
+namespace {
+
+namespace fs = std::experimental::filesystem;
+
+uint64_t
+sampleDiskUsageOnFileSystem(const fs::path &path, const HwInfo::Disk &disk)
+{
+    auto space_info = fs::space(path);
+    uint64_t result = (space_info.capacity - space_info.available);
+    if (result > disk.sizeBytes()) {
+        return disk.sizeBytes();
+    }
+    return result;
+}
+
+uint64_t
+sampleDiskUsageInDirectory(const fs::path &path)
+{
+    uint64_t result = 0;
+    for (const auto &elem : fs::recursive_directory_iterator(path,
+                                                             fs::directory_options::skip_permission_denied)) {
+        if (fs::is_regular_file(elem.path()) && !fs::is_symlink(elem.path())) {
+            result += fs::file_size(elem.path());
+        }
+    }
+    return result;
+}
+
+}
+
 void
 DiskMemUsageSampler::sampleDiskUsage()
 {
-    _filter.setDiskStats(std::experimental::filesystem::space(_path));
+    const auto &disk = _filter.getHwInfo().disk();
+    _filter.setDiskUsedSize(disk.shared() ?
+                            sampleDiskUsageInDirectory(_path) :
+                            sampleDiskUsageOnFileSystem(_path, disk));
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
index 198f111f052..4ed48613f6a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
@@ -22,19 +22,24 @@ class DiskMemUsageSampler {
     void sampleMemoryUsage();
 public:
     struct Config {
-        DiskMemUsageFilter::Config _filterConfig;
-        double _sampleInterval;
-    public:
+        DiskMemUsageFilter::Config filterConfig;
+        double sampleInterval;
+        HwInfo hwInfo;
+
         Config()
-            : _filterConfig(),
-              _sampleInterval(60.0)
+            : filterConfig(),
+              sampleInterval(60.0),
+              hwInfo()
         {
         }
 
-        Config(double memoryLimit_in, double diskLimit_in,
-               double sampleInterval_in)
-            : _filterConfig(memoryLimit_in, diskLimit_in),
-              _sampleInterval(sampleInterval_in)
+        Config(double memoryLimit_in,
+               double diskLimit_in,
+               double sampleInterval_in,
+               const HwInfo &hwInfo_in)
+            : filterConfig(memoryLimit_in, diskLimit_in),
+              sampleInterval(sampleInterval_in),
+              hwInfo(hwInfo_in)
         {
         }
     };
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index ef255315156..664808acbc0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -32,6 +32,7 @@
 #include <vespa/searchlib/expression/forcelink.hpp>
 
 #include <vespa/log/log.h>
+
 LOG_SETUP(".proton.server.proton");
 
 using document::DocumentTypeRepo;
@@ -75,12 +76,13 @@ setFS4Compression(const ProtonConfig & proton)
 }
 
 DiskMemUsageSampler::Config
-diskMemUsageSamplerConfig(const ProtonConfig &proton)
+diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo)
 {
     return DiskMemUsageSampler::Config(
             proton.writefilter.memorylimit,
             proton.writefilter.disklimit,
-            proton.writefilter.sampleinterval);
+            proton.writefilter.sampleinterval,
+            hwInfo);
 }
 
 }
@@ -238,7 +240,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
     setFS4Compression(protonConfig);
     _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
                            (protonConfig.basedir,
-                            diskMemUsageSamplerConfig(protonConfig));
+                            diskMemUsageSamplerConfig(protonConfig, _hwInfo));
 
     _metricsEngine.reset(new MetricsEngine());
     _metricsEngine->addMetricsHook(_metricsHook);
@@ -346,7 +348,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot)
                             protonConfig.search.memory.limiter.minhits);
     const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP();
 
-    _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig));
+    _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo));
     if (_memoryFlushConfigUpdater) {
         _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory);
         _flushEngine->kick();
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
index afe9f6b85d3..33b37649ed2 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
@@ -9,11 +9,10 @@ using namespace vespalib::slime;
 namespace proton {
 
 void
-convertDiskStatsToSlime(const DiskMemUsageFilter::space_info &stats, Cursor &object)
+convertDiskStatsToSlime(const HwInfo &hwInfo, uint64_t diskUsedSizeBytes, Cursor &object)
 {
-    object.setLong("capacity", stats.capacity);
-    object.setLong("free", stats.free);
-    object.setLong("available", stats.available);
+    object.setLong("capacity", hwInfo.disk().sizeBytes());
+    object.setLong("used", diskUsedSizeBytes);
 }
 
 void
@@ -39,12 +38,12 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool
         Cursor &disk = object.setObject("disk");
         disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio());
         disk.setDouble("usedLimit", config._diskLimit);
-        convertDiskStatsToSlime(_usageFilter.getDiskStats(), disk.setObject("stats"));
+        convertDiskStatsToSlime(_usageFilter.getHwInfo(), _usageFilter.getDiskUsedSize(), disk.setObject("stats"));
 
         Cursor &memory = object.setObject("memory");
         memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio());
         memory.setDouble("usedLimit", config._memoryLimit);
-        memory.setLong("physicalMemory", _usageFilter.getPhysicalMemory());
+        memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes());
         convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats"));
     } else {
         object.setDouble("disk", _usageFilter.getDiskUsedRatio());