Merge pull request #3574 from vespa-engine/geirst/add-disk-usage-sampling-for-shared-disks

Geirst/add disk usage sampling for shared disks

15 files changed, 264 insertions, 130 deletions

diff --git a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
index ecb39af61bc..ce0a6bffc0d 100644
--- a/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
+++ b/searchcore/src/tests/proton/attribute/attributeflush_test.cpp
@@ -577,7 +577,7 @@ Test::requireThatFlushedAttributeCanBeLoaded(const HwInfo &hwInfo)
 {
     constexpr uint32_t numDocs = 100;
     BaseFixture f(hwInfo);
-    vespalib::string attrName(hwInfo.slowDisk() ? "a11slow" : "a11fast");
+    vespalib::string attrName(hwInfo.disk().slow() ? "a11slow" : "a11fast");
     {
         AttributeManagerFixture amf(f);
         AttributeManager &am = amf._m;
@@ -606,8 +606,8 @@ Test::requireThatFlushedAttributeCanBeLoaded(const HwInfo &hwInfo)
 void
 Test::requireThatFlushedAttributeCanBeLoaded()
 {
-    TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(false)));
-    TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(true)));
+    TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(HwInfo::Disk(0, false, false), HwInfo::Memory(0))));
+    TEST_DO(requireThatFlushedAttributeCanBeLoaded(HwInfo(HwInfo::Disk(0, true, false), HwInfo::Memory(0))));
 }
 
 int
diff --git a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp
index f9745d7fd03..c39a8163e00 100644
--- a/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp
+++ b/searchcore/src/tests/proton/common/hw_info_sampler/hw_info_sampler_test.cpp
@@ -18,6 +18,7 @@ namespace {
 
 const vespalib::string test_dir = "temp";
 constexpr uint64_t sampleLen = 1024 * 1024 * 40;
+constexpr bool sharedDisk = false;
 
 long time_point_to_long(Clock::time_point tp)
 {
@@ -44,11 +45,11 @@ struct Fixture
 
 TEST_F("Test that hw_info_sampler uses override info", Fixture)
 {
-    Config samplerCfg(75.0, 100.0, sampleLen);
+    Config samplerCfg(0, 75.0, 100.0, sampleLen, sharedDisk, 0);
     HwInfoSampler sampler(test_dir, samplerCfg);
     EXPECT_EQUAL(75.0, sampler.diskWriteSpeed());
     EXPECT_NOT_EQUAL(0, time_point_to_long(sampler.sampleTime()));
-    EXPECT_TRUE(sampler.hwInfo().slowDisk());
+    EXPECT_TRUE(sampler.hwInfo().disk().slow());
 }
 
 TEST_F("Test that hw_info_sampler uses saved info", Fixture)
@@ -57,16 +58,16 @@ TEST_F("Test that hw_info_sampler uses saved info", Fixture)
     builder.disk.writespeed = 72.0;
     builder.disk.sampletime = time_point_to_long(Clock::now());
     f.writeConfig(builder);
-    Config samplerCfg(0.0, 70.0, sampleLen);
+    Config samplerCfg(0, 0.0, 70.0, sampleLen, sharedDisk, 0);
     HwInfoSampler sampler(test_dir, samplerCfg);
     EXPECT_EQUAL(builder.disk.writespeed, sampler.diskWriteSpeed());
     EXPECT_EQUAL(builder.disk.sampletime, time_point_to_long(sampler.sampleTime()));
-    EXPECT_FALSE(sampler.hwInfo().slowDisk());
+    EXPECT_FALSE(sampler.hwInfo().disk().slow());
 }
 
 TEST_F("Test that hw_info_sampler can sample disk write speed", Fixture)
 {
-    Config samplerCfg(0.0, 100.0, sampleLen);
+    Config samplerCfg(0, 0.0, 100.0, sampleLen, sharedDisk, 0);
     HwInfoSampler sampler(test_dir, samplerCfg);
     ASSERT_NOT_EQUAL(0.0, sampler.diskWriteSpeed());
     ASSERT_NOT_EQUAL(0, time_point_to_long(sampler.sampleTime()));
@@ -76,6 +77,34 @@ TEST_F("Test that hw_info_sampler can sample disk write speed", Fixture)
                  time_point_to_long(sampler2.sampleTime()));
 }
 
+TEST_F("require that disk size can be specified", Fixture)
+{
+    Config samplerCfg(1024, 1.0, 0.0, sampleLen, sharedDisk, 0);
+    HwInfoSampler sampler(test_dir, samplerCfg);
+    EXPECT_EQUAL(1024u, sampler.hwInfo().disk().sizeBytes());
+}
+
+TEST_F("require that disk size can be sampled", Fixture)
+{
+    Config samplerCfg(0, 1.0, 0.0, sampleLen, sharedDisk, 0);
+    HwInfoSampler sampler(test_dir, samplerCfg);
+    EXPECT_GREATER(sampler.hwInfo().disk().sizeBytes(), 0u);
+}
+
+TEST_F("require that memory size can be specified", Fixture)
+{
+    Config samplerCfg(0, 1.0, 0.0, sampleLen, sharedDisk, 1024);
+    HwInfoSampler sampler(test_dir, samplerCfg);
+    EXPECT_EQUAL(1024u, sampler.hwInfo().memory().sizeBytes());
+}
+
+TEST_F("require that memory size can be sampled", Fixture)
+{
+    Config samplerCfg(0, 1.0, 0.0, sampleLen, sharedDisk, 0);
+    HwInfoSampler sampler(test_dir, samplerCfg);
+    EXPECT_GREATER(sampler.hwInfo().memory().sizeBytes(), 0u);
+}
+
 TEST_MAIN()
 {
     vespalib::rmdir(test_dir, true);
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
index 5d8a647f675..cfd1c37e4d1 100644
--- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
@@ -1,9 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include <vespa/vespalib/testkit/testapp.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h>
 
 using proton::DiskMemUsageFilter;
+using proton::HwInfo;
 
 namespace fs = std::experimental::filesystem;
 
@@ -16,9 +18,9 @@ struct Fixture
     using Config = DiskMemUsageFilter::Config;
 
     Fixture()
-        : _filter(64 * 1024 * 1024)
+        : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(64 * 1024 * 1024)))
     {
-        _filter.setDiskStats({.capacity = 100, .free = 100, .available=100});
+        _filter.setDiskUsedSize(0);
         _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000,
                                                             10000001,
                                                             10000002,
@@ -41,7 +43,7 @@ struct Fixture
     }
 
     void triggerDiskLimit() {
-        _filter.setDiskStats({.capacity = 100, .free = 20, .available=10});
+        _filter.setDiskUsedSize(90);
     }
 
     void triggerMemoryLimit()
@@ -76,7 +78,7 @@ TEST_F("Check that disk limit can be reached", Fixture)
                 "action: \"add more content nodes\", "
                 "reason: \"disk used (0.9) > disk limit (0.8)\", "
                 "stats: { "
-                "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}");
+                "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}");
 }
 
 TEST_F("Check that memory limit can be reached", Fixture)
@@ -108,7 +110,7 @@ TEST_F("Check that both disk limit and memory limit can be reached", Fixture)
                 "action: \"add more content nodes\", "
                 "reason: \"disk used (0.9) > disk limit (0.8)\", "
                 "stats: { "
-                "capacity: 100, free: 20, available: 10, diskUsed: 0.9, diskLimit: 0.8}}");
+                "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}");
 }
 
 TEST_MAIN() { TEST_RUN_ALL(); }
diff --git a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
index 43dc3237685..a658b11263a 100644
--- a/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
+++ b/searchcore/src/vespa/searchcore/proton/attribute/flushableattribute.cpp
@@ -92,7 +92,7 @@ FlushableAttribute::Flusher::saveAttribute()
                                                  _syncToken);
     bool saveSuccess = true;
     if (_saver && _saver->hasGenerationGuard() &&
-        _fattr._hwInfo.slowDisk()) {
+        _fattr._hwInfo.disk().slow()) {
         saveSuccess = _saver->save(_saveTarget);
         _saver.reset();
     }
diff --git a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
index dec686d3070..cdca446f114 100644
--- a/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
+++ b/searchcore/src/vespa/searchcore/proton/common/CMakeLists.txt
@@ -22,4 +22,5 @@ vespa_add_library(searchcore_pcommon STATIC
     DEPENDS
     searchcore_proton_metrics
     searchcore_fconfig
+    stdc++fs
 )
diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info.h b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
index 06efaec18f2..12b2d0cfbe5 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info.h
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info.h
@@ -2,6 +2,8 @@
 
 #pragma once
 
+#include <cstdint>
+
 namespace proton {
 
 /*
@@ -9,19 +11,48 @@ namespace proton {
  */
 class HwInfo
 {
-    bool _slowDisk;
+public:
+    class Disk {
+    private:
+        uint64_t _sizeBytes;
+        bool _slow;
+        bool _shared;
+    public:
+        Disk(uint64_t sizeBytes_, bool slow_, bool shared_)
+            : _sizeBytes(sizeBytes_), _slow(slow_), _shared(shared_) {}
+        uint64_t sizeBytes() const { return _sizeBytes; }
+        bool slow() const { return _slow; }
+        bool shared() const { return _shared; }
+    };
+
+    class Memory {
+    private:
+        uint64_t _sizeBytes;
+    public:
+        Memory(uint64_t sizeBytes_) : _sizeBytes(sizeBytes_) {}
+        uint64_t sizeBytes() const { return _sizeBytes; }
+    };
+
+private:
+    Disk _disk;
+    Memory _memory;
+
 public:
     HwInfo()
-        : _slowDisk(false)
+        : _disk(0, false, false),
+          _memory(0)
     {
     }
 
-    HwInfo(bool slowDisk_in)
-        : _slowDisk(slowDisk_in)
+    HwInfo(const Disk &disk_,
+           const Memory &memory_)
+        : _disk(disk_),
+          _memory(memory_)
     {
     }
 
-    bool slowDisk() const { return _slowDisk; }
+    const Disk &disk() const { return _disk; }
+    const Memory &memory() const { return _memory; }
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp
index 73bb20c712a..ccb88714a20 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.cpp
@@ -1,13 +1,14 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 
 #include "hw_info_sampler.h"
-#include <vespa/config/config.h>
 #include <vespa/config/common/configholder.h>
+#include <vespa/config/config.h>
 #include <vespa/config/file/filesource.h>
-#include <vespa/searchcore/config/config-hwinfo.h>
 #include <vespa/config/print/fileconfigwriter.h>
-#include <vespa/vespalib/io/fileutil.h>
 #include <vespa/fastos/file.h>
+#include <vespa/searchcore/config/config-hwinfo.h>
+#include <vespa/vespalib/io/fileutil.h>
+#include <experimental/filesystem>
 
 using config::ConfigHandle;
 using config::ConfigSubscriber;
@@ -22,6 +23,26 @@ namespace proton {
 
 namespace {
 
+uint64_t
+sampleDiskSizeBytes(const std::string &pathStr, const HwInfoSampler::Config &cfg)
+{
+    if (cfg.diskSizeBytes != 0) {
+        return cfg.diskSizeBytes;
+    }
+    std::experimental::filesystem::path path(pathStr);
+    auto space_info = std::experimental::filesystem::space(path);
+    return space_info.capacity;
+}
+
+uint64_t
+sampleMemorySizeBytes(const HwInfoSampler::Config &cfg)
+{
+    if (cfg.memorySizeBytes != 0) {
+        return cfg.memorySizeBytes;
+    }
+    return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
+}
+
 std::unique_ptr<HwinfoConfig> readConfig(const vespalib::string &path) {
     FileSpec spec(path + "/" + "hwinfo.cfg");
     ConfigSubscriber s(spec);
@@ -80,8 +101,29 @@ HwInfoSampler::HwInfoSampler(const vespalib::string &path,
       _sampleTime(),
       _diskWriteSpeed(0.0)
 {
-    if (config._diskWriteSpeedOverride != 0) {
-        _diskWriteSpeed = config._diskWriteSpeedOverride;
+    setDiskWriteSpeed(path, config);
+    setup(HwInfo::Disk(sampleDiskSizeBytes(path, config),
+                       (_diskWriteSpeed < config.slowWriteSpeedLimit),
+                       config.diskShared),
+          HwInfo::Memory(sampleMemorySizeBytes(config)));
+
+}
+
+HwInfoSampler::~HwInfoSampler()
+{
+}
+
+void
+HwInfoSampler::setup(const HwInfo::Disk &disk, const HwInfo::Memory &memory)
+{
+    _hwInfo = HwInfo(disk, memory);
+}
+
+void
+HwInfoSampler::setDiskWriteSpeed(const vespalib::string &path, const Config &config)
+{
+    if (config.diskWriteSpeedOverride != 0) {
+        _diskWriteSpeed = config.diskWriteSpeedOverride;
         _sampleTime = Clock::now();
     } else {
         auto cfg = readConfig(path);
@@ -89,28 +131,16 @@ HwInfoSampler::HwInfoSampler(const vespalib::string &path,
             _sampleTime = std::chrono::time_point<Clock>(std::chrono::seconds(cfg->disk.sampletime));
             _diskWriteSpeed = cfg->disk.writespeed;
         } else {
-            sample(path, config);
+            sampleDiskWriteSpeed(path, config);
         }
     }
-    setup(config);
-}
-
-HwInfoSampler::~HwInfoSampler()
-{
-}
-
-void
-HwInfoSampler::setup(const Config &config)
-{
-    bool slowDisk = _diskWriteSpeed < config._slowWriteSpeedLimit;
-    _hwInfo = HwInfo(slowDisk);
 }
 
 void
-HwInfoSampler::sample(const vespalib::string &path, const Config &config)
+HwInfoSampler::sampleDiskWriteSpeed(const vespalib::string &path, const Config &config)
 {
     size_t minDiskWriteLen = 1024u * 1024u;
-    size_t diskWriteLen = config._diskSampleWriteSize;
+    size_t diskWriteLen = config.diskSampleWriteSize;
     diskWriteLen = std::max(diskWriteLen, minDiskWriteLen);
     _sampleTime = Clock::now();
     _diskWriteSpeed = measureDiskWriteSpeed(path, diskWriteLen);
diff --git a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h
index 22af2d32786..b640fd50370 100644
--- a/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h
+++ b/searchcore/src/vespa/searchcore/proton/common/hw_info_sampler.h
@@ -16,16 +16,25 @@ class HwInfoSampler
 {
 public:
     struct Config {
-        double _diskWriteSpeedOverride;
-        double _slowWriteSpeedLimit;
-        uint64_t _diskSampleWriteSize;
-
-        Config(double diskWriteSpeedOverride,
-               double slowWriteSpeedLimit,
-               double diskSampleWriteSize)
-            : _diskWriteSpeedOverride(diskWriteSpeedOverride),
-              _slowWriteSpeedLimit(slowWriteSpeedLimit),
-              _diskSampleWriteSize(diskSampleWriteSize)
+        uint64_t diskSizeBytes;
+        double diskWriteSpeedOverride;
+        double slowWriteSpeedLimit;
+        uint64_t diskSampleWriteSize;
+        bool diskShared;
+        uint64_t memorySizeBytes;
+
+        Config(uint64_t diskSizeBytes_,
+               double diskWriteSpeedOverride_,
+               double slowWriteSpeedLimit_,
+               double diskSampleWriteSize_,
+               bool diskShared_,
+               uint64_t memorySizeBytes_)
+            : diskSizeBytes(diskSizeBytes_),
+              diskWriteSpeedOverride(diskWriteSpeedOverride_),
+              slowWriteSpeedLimit(slowWriteSpeedLimit_),
+              diskSampleWriteSize(diskSampleWriteSize_),
+              diskShared(diskShared_),
+              memorySizeBytes(memorySizeBytes_)
         {
         }
     };
@@ -36,8 +45,9 @@ private:
     Clock::time_point _sampleTime;
     double _diskWriteSpeed;
 
-    void setup(const Config &config);
-    void sample(const vespalib::string &path, const Config &config);
+    void setup(const HwInfo::Disk &disk, const HwInfo::Memory &memory);
+    void setDiskWriteSpeed(const vespalib::string &path, const Config &config);
+    void sampleDiskWriteSpeed(const vespalib::string &path, const Config &config);
 public:
     HwInfoSampler(const vespalib::string &path, const Config &config);
     ~HwInfoSampler();
diff --git a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
index 322e71a572c..3821237233e 100644
--- a/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
+++ b/searchcore/src/vespa/searchcore/proton/documentmetastore/documentmetastoreflushtarget.cpp
@@ -87,7 +87,7 @@ DocumentMetaStoreFlushTarget::Flusher::saveDocumentMetaStore()
     SerialNumFileHeaderContext fileHeaderContext(_dmsft._fileHeaderContext,
                                                  _syncToken);
     bool saveSuccess = false;
-    if (_dmsft._hwInfo.slowDisk()) {
+    if (_dmsft._hwInfo.disk().slow()) {
         search::AttributeMemorySaveTarget memorySaveTarget;
         saveSuccess = _saver->save(memorySaveTarget);
         _saver.reset();
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index d489f477df2..32b2b6aaba6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -3,6 +3,7 @@
 #include "disk_mem_usage_filter.h"
 #include "i_disk_mem_usage_listener.h"
 #include <vespa/log/log.h>
+#include <vespa/searchcore/proton/common/hw_info.h>
 
 LOG_SETUP(".proton.server.disk_mem_usage_filter");
 
@@ -43,12 +44,12 @@ void
 makeDiskStatsMessage(std::ostream &os,
                      double diskUsed,
                      double diskLimit,
-                     const DiskMemUsageFilter::space_info &diskStats)
+                     const HwInfo &hwInfo,
+                     uint64_t usedDiskSizeBytes)
 {
     os << "stats: { ";
-    os << "capacity: " << diskStats.capacity << ", ";
-    os << "free: " << diskStats.free << ", ";
-    os << "available: " << diskStats.available << ", ";
+    os << "capacity: " << hwInfo.disk().sizeBytes() << ", ";
+    os << "used: " << usedDiskSizeBytes << ", ";
     os << "diskUsed: " << diskUsed << ", ";
     os << "diskLimit: " << diskLimit << "}";
 }
@@ -57,31 +58,31 @@ void
 makeDiskLimitMessage(std::ostream &os,
                      double diskUsed,
                      double diskLimit,
-                     const DiskMemUsageFilter::space_info &diskStats)
+                     const HwInfo &hwInfo,
+                     uint64_t usedDiskSizeBytes)
 {
     os << "diskLimitReached: { ";
     os << "action: \"add more content nodes\", ";
     os << "reason: \"disk used (" << diskUsed << ") > disk limit (" << diskLimit << ")\", ";
-    makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats);
+    makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes);
     os << "}";
 }
 
-
 vespalib::string
 makeUnblockingMessage(double memoryUsed,
                       double memoryLimit,
                       const vespalib::ProcessMemoryStats &memoryStats,
-                      uint64_t physicalMemory,
+                      const HwInfo &hwInfo,
                       double diskUsed,
                       double diskLimit,
-                      const DiskMemUsageFilter::space_info &diskStats)
+                      uint64_t usedDiskSizeBytes)
 {
     std::ostringstream os;
     os << "memoryLimitOK: { ";
-    makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, physicalMemory);
+    makeMemoryStatsMessage(os, memoryUsed, memoryLimit, memoryStats, hwInfo.memory().sizeBytes());
     os << "}, ";
     os << "diskLimitOK: { ";
-    makeDiskStatsMessage(os, diskUsed, diskLimit, diskStats);
+    makeDiskStatsMessage(os, diskUsed, diskLimit, hwInfo, usedDiskSizeBytes);
     os << "}";
     return os.str();
 }
@@ -97,7 +98,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
     if (memoryUsed > _config._memoryLimit) {
         hasMessage = true;
         makeMemoryLimitMessage(message, memoryUsed,
-                _config._memoryLimit, _memoryStats, _physicalMemory);
+                _config._memoryLimit, _memoryStats, _hwInfo.memory().sizeBytes());
     }
     double diskUsed = getDiskUsedRatio(guard);
     if (diskUsed > _config._diskLimit) {
@@ -105,7 +106,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
             message << ", ";
         }
         hasMessage = true;
-        makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _diskStats);
+        makeDiskLimitMessage(message, diskUsed, _config._diskLimit, _hwInfo, _diskUsedSizeBytes);
     }
     if (hasMessage) {
         if (_acceptWrite) {
@@ -118,10 +119,10 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
             vespalib::string unblockMsg = makeUnblockingMessage(memoryUsed,
                                                                 _config._memoryLimit,
                                                                 _memoryStats,
-                                                                _physicalMemory,
+                                                                _hwInfo,
                                                                 diskUsed,
                                                                 _config._diskLimit,
-                                                                _diskStats);
+                                                                _diskUsedSizeBytes);
             LOG(info, "Write operations are now un-blocked: '%s'", unblockMsg.c_str());
         }
         _state = State();
@@ -137,23 +138,23 @@ DiskMemUsageFilter::getMemoryUsedRatio(const Guard &guard) const
 {
     (void) guard;
     uint64_t unscaledMemoryUsed = _memoryStats.getAnonymousRss();
-    return static_cast<double>(unscaledMemoryUsed) / _physicalMemory;
+    return static_cast<double>(unscaledMemoryUsed) / _hwInfo.memory().sizeBytes();
 }
 
 double
 DiskMemUsageFilter::getDiskUsedRatio(const Guard &guard) const
 {
     (void) guard;
-    double availableDiskSpaceRatio = static_cast<double>(_diskStats.available) /
-                                     static_cast<double>(_diskStats.capacity);
-    return 1.0 - availableDiskSpaceRatio;
+    double usedDiskSpaceRatio = static_cast<double>(_diskUsedSizeBytes) /
+                                static_cast<double>(_hwInfo.disk().sizeBytes());
+    return usedDiskSpaceRatio;
 }
 
-DiskMemUsageFilter::DiskMemUsageFilter(uint64_t physicalMemory_in)
+DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
     : _lock(),
+      _hwInfo(hwInfo),
       _memoryStats(),
-      _physicalMemory(physicalMemory_in),
-      _diskStats(),
+      _diskUsedSizeBytes(),
       _config(),
       _state(),
       _acceptWrite(true),
@@ -172,10 +173,10 @@ DiskMemUsageFilter::setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in)
 }
 
 void
-DiskMemUsageFilter::setDiskStats(space_info diskStats_in)
+DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes)
 {
     Guard guard(_lock);
-    _diskStats = diskStats_in;
+    _diskUsedSizeBytes = diskUsedSizeBytes;
     recalcState(guard);
 }
 
@@ -194,11 +195,11 @@ DiskMemUsageFilter::getMemoryStats() const
     return _memoryStats;
 }
 
-DiskMemUsageFilter::space_info
-DiskMemUsageFilter::getDiskStats() const
+uint64_t
+DiskMemUsageFilter::getDiskUsedSize() const
 {
     Guard guard(_lock);
-    return _diskStats;
+    return _diskUsedSizeBytes;
 }
 
 DiskMemUsageFilter::Config
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
index c06d2e2ab0a..4906348a54d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -4,12 +4,12 @@
 
 #include "i_disk_mem_usage_notifier.h"
 #include "disk_mem_usage_state.h"
+#include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
 #include <vespa/vespalib/util/process_memory_stats.h>
-#include <mutex>
 #include <atomic>
 #include <experimental/filesystem>
-
+#include <mutex>
 
 namespace proton {
 
@@ -39,10 +39,10 @@ public:
     };
 
 private:
-    mutable Mutex _lock; // protect _memoryStats, _diskStats, _config, _state
+    mutable Mutex _lock; // protect _memoryStats, _usedDiskSizeBytes, _config, _state
+    HwInfo                       _hwInfo;
     vespalib::ProcessMemoryStats _memoryStats;
-    uint64_t                     _physicalMemory;
-    space_info                   _diskStats;
+    uint64_t                     _diskUsedSizeBytes;
     Config                       _config;
     State                        _state;
     std::atomic<bool>            _acceptWrite;
@@ -55,15 +55,15 @@ private:
     void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state);
 
 public:
-    DiskMemUsageFilter(uint64_t physicalMememory_in);
+    DiskMemUsageFilter(const HwInfo &hwInfo);
     ~DiskMemUsageFilter();
     void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in);
-    void setDiskStats(space_info diskStats_in);
+    void setDiskUsedSize(uint64_t diskUsedSizeBytes);
     void setConfig(Config config);
     vespalib::ProcessMemoryStats getMemoryStats() const;
-    space_info getDiskStats() const;
+    uint64_t getDiskUsedSize() const;
     Config getConfig() const;
-    uint64_t getPhysicalMemory() const { return _physicalMemory; }
+    const HwInfo &getHwInfo() const { return _hwInfo; }
     double getMemoryUsedRatio() const;
     double getDiskUsedRatio() const;
     bool acceptWriteOperation() const override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
index bacf80e69a6..ddbede13880 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
@@ -3,30 +3,16 @@
 #include "disk_mem_usage_sampler.h"
 #include <vespa/vespalib/util/timer.h>
 #include <vespa/vespalib/util/lambdatask.h>
+#include <experimental/filesystem>
 #include <unistd.h>
 
 using vespalib::makeLambdaTask;
 
 namespace proton {
 
-namespace {
-
-uint64_t getPhysicalMemoryBytes()
-{
-    // TODO: Temporal workaround for Docker nodes. Remove when this is part of proton.cfg instead.
-    if (const char *memoryEnv = std::getenv("VESPA_TOTAL_MEMORY_MB")) {
-        uint64_t physicalMemoryMB = atoll(memoryEnv);
-        return physicalMemoryMB * 1024u * 1024u;
-    } else {
-        return sysconf(_SC_PHYS_PAGES) * sysconf(_SC_PAGESIZE);
-    }
-}
-
-} // namespace proton:<anonymous>
-
 DiskMemUsageSampler::DiskMemUsageSampler(const std::string &path_in,
                                          const Config &config)
-    : _filter(getPhysicalMemoryBytes()),
+    : _filter(config.hwInfo),
       _path(path_in),
       _sampleInterval(60.0),
       _periodicTimer()
@@ -43,8 +29,8 @@ void
 DiskMemUsageSampler::setConfig(const Config &config)
 {
     _periodicTimer.reset();
-    _filter.setConfig(config._filterConfig);
-    _sampleInterval = config._sampleInterval;
+    _filter.setConfig(config.filterConfig);
+    _sampleInterval = config.sampleInterval;
     sampleUsage();
     _periodicTimer = std::make_unique<vespalib::Timer>();
     _periodicTimer->scheduleAtFixedRate(makeLambdaTask([this]()
@@ -59,10 +45,43 @@ DiskMemUsageSampler::sampleUsage()
     sampleDiskUsage();
 }
 
+namespace {
+
+namespace fs = std::experimental::filesystem;
+
+uint64_t
+sampleDiskUsageOnFileSystem(const fs::path &path, const HwInfo::Disk &disk)
+{
+    auto space_info = fs::space(path);
+    uint64_t result = (space_info.capacity - space_info.available);
+    if (result > disk.sizeBytes()) {
+        return disk.sizeBytes();
+    }
+    return result;
+}
+
+uint64_t
+sampleDiskUsageInDirectory(const fs::path &path)
+{
+    uint64_t result = 0;
+    for (const auto &elem : fs::recursive_directory_iterator(path,
+                                                             fs::directory_options::skip_permission_denied)) {
+        if (fs::is_regular_file(elem.path()) && !fs::is_symlink(elem.path())) {
+            result += fs::file_size(elem.path());
+        }
+    }
+    return result;
+}
+
+}
+
 void
 DiskMemUsageSampler::sampleDiskUsage()
 {
-    _filter.setDiskStats(std::experimental::filesystem::space(_path));
+    const auto &disk = _filter.getHwInfo().disk();
+    _filter.setDiskUsedSize(disk.shared() ?
+                            sampleDiskUsageInDirectory(_path) :
+                            sampleDiskUsageOnFileSystem(_path, disk));
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
index 198f111f052..4ed48613f6a 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.h
@@ -22,19 +22,24 @@ class DiskMemUsageSampler {
     void sampleMemoryUsage();
 public:
     struct Config {
-        DiskMemUsageFilter::Config _filterConfig;
-        double _sampleInterval;
-    public:
+        DiskMemUsageFilter::Config filterConfig;
+        double sampleInterval;
+        HwInfo hwInfo;
+
         Config()
-            : _filterConfig(),
-              _sampleInterval(60.0)
+            : filterConfig(),
+              sampleInterval(60.0),
+              hwInfo()
         {
         }
 
-        Config(double memoryLimit_in, double diskLimit_in,
-               double sampleInterval_in)
-            : _filterConfig(memoryLimit_in, diskLimit_in),
-              _sampleInterval(sampleInterval_in)
+        Config(double memoryLimit_in,
+               double diskLimit_in,
+               double sampleInterval_in,
+               const HwInfo &hwInfo_in)
+            : filterConfig(memoryLimit_in, diskLimit_in),
+              sampleInterval(sampleInterval_in),
+              hwInfo(hwInfo_in)
         {
         }
     };
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index d95b0fd44d1..664808acbc0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -32,6 +32,7 @@
 #include <vespa/searchlib/expression/forcelink.hpp>
 
 #include <vespa/log/log.h>
+
 LOG_SETUP(".proton.server.proton");
 
 using document::DocumentTypeRepo;
@@ -75,12 +76,13 @@ setFS4Compression(const ProtonConfig & proton)
 }
 
 DiskMemUsageSampler::Config
-diskMemUsageSamplerConfig(const ProtonConfig &proton)
+diskMemUsageSamplerConfig(const ProtonConfig &proton, const HwInfo &hwInfo)
 {
     return DiskMemUsageSampler::Config(
             proton.writefilter.memorylimit,
             proton.writefilter.disklimit,
-            proton.writefilter.sampleinterval);
+            proton.writefilter.sampleinterval,
+            hwInfo);
 }
 
 }
@@ -223,17 +225,22 @@ Proton::init(const BootstrapConfig::SP & configSnapshot)
 {
     assert( _initStarted && ! _initComplete );
     const ProtonConfig &protonConfig = configSnapshot->getProtonConfig();
-    const auto &samplerCfgArgs = protonConfig.hwinfo.disk;
-    HwInfoSampler::Config samplerCfg(samplerCfgArgs.writespeed,
-                                     samplerCfgArgs.slowwritespeedlimit,
-                                     samplerCfgArgs.samplewritesize);
+    const auto &hwDiskCfg = protonConfig.hwinfo.disk;
+    const auto &hwMemoryCfg = protonConfig.hwinfo.memory;
+    // TODO: Forward disk size when performance impact of disk usage sampling is verified
+    HwInfoSampler::Config samplerCfg(0,
+                                     hwDiskCfg.writespeed,
+                                     hwDiskCfg.slowwritespeedlimit,
+                                     hwDiskCfg.samplewritesize,
+                                     hwDiskCfg.shared,
+                                     hwMemoryCfg.size);
     _hwInfoSampler = std::make_unique<HwInfoSampler>(protonConfig.basedir,
                                                      samplerCfg);
     _hwInfo = _hwInfoSampler->hwInfo();
     setFS4Compression(protonConfig);
     _diskMemUsageSampler = std::make_unique<DiskMemUsageSampler>
                            (protonConfig.basedir,
-                            diskMemUsageSamplerConfig(protonConfig));
+                            diskMemUsageSamplerConfig(protonConfig, _hwInfo));
 
     _metricsEngine.reset(new MetricsEngine());
     _metricsEngine->addMetricsHook(_metricsHook);
@@ -341,7 +348,7 @@ Proton::applyConfig(const BootstrapConfig::SP & configSnapshot)
                             protonConfig.search.memory.limiter.minhits);
     const DocumentTypeRepo::SP repo = configSnapshot->getDocumentTypeRepoSP();
 
-    _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig));
+    _diskMemUsageSampler->setConfig(diskMemUsageSamplerConfig(protonConfig, _hwInfo));
     if (_memoryFlushConfigUpdater) {
         _memoryFlushConfigUpdater->setConfig(protonConfig.flush.memory);
         _flushEngine->kick();
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
index afe9f6b85d3..33b37649ed2 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
@@ -9,11 +9,10 @@ using namespace vespalib::slime;
 namespace proton {
 
 void
-convertDiskStatsToSlime(const DiskMemUsageFilter::space_info &stats, Cursor &object)
+convertDiskStatsToSlime(const HwInfo &hwInfo, uint64_t diskUsedSizeBytes, Cursor &object)
 {
-    object.setLong("capacity", stats.capacity);
-    object.setLong("free", stats.free);
-    object.setLong("available", stats.available);
+    object.setLong("capacity", hwInfo.disk().sizeBytes());
+    object.setLong("used", diskUsedSizeBytes);
 }
 
 void
@@ -39,12 +38,12 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool
         Cursor &disk = object.setObject("disk");
         disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio());
         disk.setDouble("usedLimit", config._diskLimit);
-        convertDiskStatsToSlime(_usageFilter.getDiskStats(), disk.setObject("stats"));
+        convertDiskStatsToSlime(_usageFilter.getHwInfo(), _usageFilter.getDiskUsedSize(), disk.setObject("stats"));
 
         Cursor &memory = object.setObject("memory");
         memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio());
         memory.setDouble("usedLimit", config._memoryLimit);
-        memory.setLong("physicalMemory", _usageFilter.getPhysicalMemory());
+        memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes());
         convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats"));
     } else {
         object.setDouble("disk", _usageFilter.getDiskUsedRatio());