Prepare for tracking the current transient disk usage across all document dbs.

The next step will be sampling the total disk usage of all current fusion indexes
and report this as the transient disk usage.

12 files changed, 80 insertions, 55 deletions

diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
index 072b76c6443..ce85517ee09 100644
--- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
@@ -121,9 +121,16 @@ TEST_F(DiskMemUsageFilterTest, both_disk_limit_and_memory_limit_can_be_reached)
               "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}");
 }
 
+TEST_F(DiskMemUsageFilterTest, transient_disk_usage_is_tracked_in_usage_state_and_metrics)
+{
+    _filter.set_transient_resource_usage({40, 0});
+    EXPECT_EQ(0.4, _filter.usageState().transient_disk_usage());
+    EXPECT_EQ(0.4, _filter.get_metrics().get_transient_disk_usage());
+}
+
 TEST_F(DiskMemUsageFilterTest, transient_memory_usage_is_tracked_in_usage_state_and_metrics)
 {
-    _filter.set_transient_resource_usage(200, 0);
+    _filter.set_transient_resource_usage({0, 200});
     EXPECT_EQ(0.2, _filter.usageState().transient_memory_usage());
     EXPECT_EQ(0.2, _filter.get_metrics().get_transient_memory_usage());
 }
diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp
index 7fa51bb8b95..0c80553e1e7 100644
--- a/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_sampler/disk_mem_usage_sampler_test.cpp
@@ -34,8 +34,7 @@ public:
         : _memory_usage(memory_usage),
           _disk_usage(disk_usage)
     {}
-    size_t get_transient_memory_usage() const override { return _memory_usage; }
-    size_t get_transient_disk_usage() const override { return _disk_usage; }
+    TransientResourceUsage get_transient_resource_usage() const override { return {_disk_usage, _memory_usage}; }
 };
 
 struct DiskMemUsageSamplerTest : public ::testing::Test {
@@ -46,7 +45,7 @@ struct DiskMemUsageSamplerTest : public ::testing::Test {
                                             50ms, make_hw_info()))
     {
         sampler.add_transient_usage_provider(std::make_shared<MyProvider>(50, 200));
-        sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 199));
+        sampler.add_transient_usage_provider(std::make_shared<MyProvider>(100, 150));
     }
     const DiskMemUsageFilter& filter() const { return sampler.writeFilter(); }
 };
@@ -56,7 +55,7 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled)
     // Poll for up to 20 seconds to get a sample.
     size_t i = 0;
     for (; i < static_cast<size_t>(20s / 50ms); ++i) {
-        if (filter().get_transient_memory_usage() > 0) {
+        if (filter().get_transient_resource_usage().memory() > 0) {
             break;
         }
         std::this_thread::sleep_for(50ms);
@@ -70,10 +69,10 @@ TEST_F(DiskMemUsageSamplerTest, resource_usage_is_sampled)
     EXPECT_EQ(filter().getMemoryStats().getAnonymousRss(), 0);
 #endif
     EXPECT_GT(filter().getDiskUsedSize(), 0);
-    EXPECT_EQ(150, filter().get_transient_memory_usage());
+    EXPECT_EQ(150, filter().get_transient_resource_usage().memory());
     EXPECT_EQ(150.0 / memory_size_bytes, filter().usageState().transient_memory_usage());
-    EXPECT_EQ(200, filter().get_transient_disk_usage());
-    EXPECT_EQ(200.0 / disk_size_bytes, filter().get_relative_transient_disk_usage());
+    EXPECT_EQ(350, filter().get_transient_resource_usage().disk());
+    EXPECT_EQ(350.0 / disk_size_bytes, filter().usageState().transient_disk_usage());
 }
 
 GTEST_MAIN_RUN_ALL_TESTS()
diff --git a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h
index 5aeb84feb2c..833b6519046 100644
--- a/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h
+++ b/searchcore/src/vespa/searchcore/proton/common/i_transient_resource_usage_provider.h
@@ -7,6 +7,32 @@
 namespace proton {
 
 /**
+ * Class containing transient disk and memory usage (in bytes).
+ */
+class TransientResourceUsage {
+private:
+    size_t _disk;
+    size_t _memory;
+
+public:
+    TransientResourceUsage() noexcept
+        : _disk(0),
+          _memory(0)
+    {}
+    TransientResourceUsage(size_t disk_in,
+                           size_t memory_in) noexcept
+        : _disk(disk_in),
+          _memory(memory_in)
+    {}
+    size_t disk() const noexcept { return _disk; }
+    size_t memory() const noexcept { return _memory; }
+    void merge(const TransientResourceUsage& rhs) {
+        _disk += rhs.disk();
+        _memory += rhs.memory();
+    }
+};
+
+/**
  * Interface class providing a snapshot of transient resource usage.
  *
  * E.g. the memory used by the memory index and extra disk needed for running disk index fusion.
@@ -15,8 +41,7 @@ namespace proton {
 class ITransientResourceUsageProvider {
 public:
     virtual ~ITransientResourceUsageProvider() = default;
-    virtual size_t get_transient_memory_usage() const = 0;
-    virtual size_t get_transient_disk_usage() const = 0;
+    virtual TransientResourceUsage get_transient_resource_usage() const = 0;
 };
 
 }
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index 27eb636b475..8928045b814 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -131,6 +131,7 @@ DiskMemUsageFilter::recalcState(const Guard &guard)
     }
     DiskMemUsageState dmstate(ResourceUsageState(_config._diskLimit, diskUsed),
                               ResourceUsageState(_config._memoryLimit, memoryUsed),
+                              get_relative_transient_disk_usage(guard),
                               get_relative_transient_memory_usage(guard));
     notifyDiskMemUsage(guard, dmstate);
 }
@@ -153,9 +154,14 @@ DiskMemUsageFilter::getDiskUsedRatio(const Guard&) const
 double
 DiskMemUsageFilter::get_relative_transient_memory_usage(const Guard&) const
 {
-    return  static_cast<double>(_transient_memory_usage) / _hwInfo.memory().sizeBytes();
+    return  static_cast<double>(_transient_usage.memory()) / _hwInfo.memory().sizeBytes();
 }
 
+double
+DiskMemUsageFilter::get_relative_transient_disk_usage(const Guard&) const
+{
+    return  static_cast<double>(_transient_usage.disk()) / _hwInfo.disk().sizeBytes();
+}
 
 DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
     : _lock(),
@@ -163,7 +169,7 @@ DiskMemUsageFilter::DiskMemUsageFilter(const HwInfo &hwInfo)
       _acceptWrite(true),
       _memoryStats(),
       _diskUsedSizeBytes(),
-      _transient_memory_usage(0u),
+      _transient_usage(),
       _config(),
       _state(),
       _dmstate(),
@@ -190,11 +196,10 @@ DiskMemUsageFilter::setDiskUsedSize(uint64_t diskUsedSizeBytes)
 }
 
 void
-DiskMemUsageFilter::set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage)
+DiskMemUsageFilter::set_transient_resource_usage(const TransientResourceUsage& transient_usage)
 {
     Guard guard(_lock);
-    _transient_memory_usage = transient_memory_usage;
-    _transient_disk_usage = transient_disk_usage;
+    _transient_usage = transient_usage;
     recalcState(guard);
 }
 
@@ -220,25 +225,11 @@ DiskMemUsageFilter::getDiskUsedSize() const
     return _diskUsedSizeBytes;
 }
 
-size_t
-DiskMemUsageFilter::get_transient_memory_usage() const
-{
-    Guard guard(_lock);
-    return _transient_memory_usage;
-}
-
-size_t
-DiskMemUsageFilter::get_transient_disk_usage() const
-{
-    Guard guard(_lock);
-    return _transient_disk_usage;
-}
-
-double
-DiskMemUsageFilter::get_relative_transient_disk_usage() const
+TransientResourceUsage
+DiskMemUsageFilter::get_transient_resource_usage() const
 {
     Guard guard(_lock);
-    return  static_cast<double>(_transient_disk_usage) / _hwInfo.disk().sizeBytes();
+    return _transient_usage;
 }
 
 DiskMemUsageFilter::Config
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
index fb18a12084f..ac328c3ddea 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -6,6 +6,7 @@
 #include "disk_mem_usage_state.h"
 #include "disk_mem_usage_metrics.h"
 #include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/searchcore/proton/common/i_transient_resource_usage_provider.h>
 #include <vespa/searchcore/proton/persistenceengine/i_resource_write_filter.h>
 #include <vespa/vespalib/util/process_memory_stats.h>
 #include <atomic>
@@ -47,8 +48,7 @@ private:
     // Following member variables are protected by _lock
     vespalib::ProcessMemoryStats _memoryStats;
     uint64_t                     _diskUsedSizeBytes;
-    size_t                       _transient_memory_usage;
-    size_t                       _transient_disk_usage;
+    TransientResourceUsage       _transient_usage;
     Config                       _config;
     State                        _state;
     DiskMemUsageState            _dmstate;
@@ -59,6 +59,7 @@ private:
     double getMemoryUsedRatio(const Guard &guard) const;
     double getDiskUsedRatio(const Guard &guard) const;
     double get_relative_transient_memory_usage(const Guard& guard) const;
+    double get_relative_transient_disk_usage(const Guard& guard) const;
     void notifyDiskMemUsage(const Guard &guard, DiskMemUsageState state);
 
 public:
@@ -66,13 +67,11 @@ public:
     ~DiskMemUsageFilter() override;
     void setMemoryStats(vespalib::ProcessMemoryStats memoryStats_in);
     void setDiskUsedSize(uint64_t diskUsedSizeBytes);
-    void set_transient_resource_usage(size_t transient_memory_usage, size_t transient_disk_usage);
+    void set_transient_resource_usage(const TransientResourceUsage& transient_usage);
     void setConfig(Config config);
     vespalib::ProcessMemoryStats getMemoryStats() const;
     uint64_t getDiskUsedSize() const;
-    size_t get_transient_memory_usage() const;
-    size_t get_transient_disk_usage() const;
-    double get_relative_transient_disk_usage() const;
+    TransientResourceUsage get_transient_resource_usage() const;
     Config getConfig() const;
     const HwInfo &getHwInfo() const { return _hwInfo; }
     DiskMemUsageState usageState() const;
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp
index 562e20a864c..230593c2c1d 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.cpp
@@ -14,6 +14,7 @@ DiskMemUsageMetrics::DiskMemUsageMetrics() noexcept
 DiskMemUsageMetrics::DiskMemUsageMetrics(const DiskMemUsageState &usage_state) noexcept
     : _disk_usage(usage_state.diskState().usage()),
       _disk_utilization(usage_state.diskState().utilization()),
+      _transient_disk_usage(usage_state.transient_disk_usage()),
       _memory_usage(usage_state.memoryState().usage()),
       _memory_utilization(usage_state.memoryState().utilization()),
       _transient_memory_usage(usage_state.transient_memory_usage())
@@ -25,6 +26,7 @@ DiskMemUsageMetrics::merge(const DiskMemUsageState &usage_state) noexcept
 {
     _disk_usage = std::max(_disk_usage, usage_state.diskState().usage());
     _disk_utilization = std::max(_disk_utilization, usage_state.diskState().utilization());
+    _transient_disk_usage = std::max(_transient_disk_usage, usage_state.transient_disk_usage());
     _memory_usage = std::max(_memory_usage, usage_state.memoryState().usage());
     _memory_utilization = std::max(_memory_utilization, usage_state.memoryState().utilization());
     _transient_memory_usage = std::max(_transient_memory_usage, usage_state.transient_memory_usage());
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h
index cb084b90a11..cb97eb4c891 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_metrics.h
@@ -14,6 +14,7 @@ class DiskMemUsageMetrics
 {
     double _disk_usage;
     double _disk_utilization;
+    double _transient_disk_usage;
     double _memory_usage;
     double _memory_utilization;
     double _transient_memory_usage;
@@ -24,6 +25,7 @@ public:
     void merge(const DiskMemUsageState &usage_state) noexcept;
     double get_disk_usage() const noexcept { return _disk_usage; }
     double get_disk_utilization() const noexcept { return _disk_utilization; }
+    double get_transient_disk_usage() const noexcept { return _transient_disk_usage; }
     double get_memory_usage() const noexcept { return _memory_usage; }
     double get_memory_utilization() const noexcept { return _memory_utilization; }
     double get_transient_memory_usage() const noexcept { return _transient_memory_usage; }
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
index cfed4c0522a..2dc749ce26b 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_sampler.cpp
@@ -125,16 +125,14 @@ DiskMemUsageSampler::sampleMemoryUsage()
 void
 DiskMemUsageSampler::sample_transient_resource_usage()
 {
-    size_t transient_memory_usage_sum = 0;
-    size_t max_transient_disk_usage = 0;
+    TransientResourceUsage transient_usage;
     {
         std::lock_guard<std::mutex> guard(_lock);
         for (auto provider : _transient_usage_providers) {
-            transient_memory_usage_sum += provider->get_transient_memory_usage();
-            max_transient_disk_usage = std::max(max_transient_disk_usage, provider->get_transient_disk_usage());
+            transient_usage.merge(provider->get_transient_resource_usage());
         }
     }
-    _filter.set_transient_resource_usage(transient_memory_usage_sum, max_transient_disk_usage);
+    _filter.set_transient_resource_usage(transient_usage);
 }
 
 void
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h
index 769086574f0..b205b441bcf 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_state.h
@@ -8,27 +8,31 @@ namespace proton {
 
 /**
  * Class used to describe state of disk and memory usage relative to configured limits.
- * In addition relative transient memory usage is tracked.
+ * In addition relative transient disk and memory usage are tracked.
  */
 class DiskMemUsageState
 {
     ResourceUsageState _diskState;
     ResourceUsageState _memoryState;
+    double _transient_disk_usage;
     double _transient_memory_usage;
 
 public:
     DiskMemUsageState() = default;
     DiskMemUsageState(const ResourceUsageState &diskState_,
                       const ResourceUsageState &memoryState_,
+                      double transient_disk_usage_ = 0,
                       double transient_memory_usage_ = 0)
         : _diskState(diskState_),
           _memoryState(memoryState_),
+          _transient_disk_usage(transient_disk_usage_),
           _transient_memory_usage(transient_memory_usage_)
     {
     }
     bool operator==(const DiskMemUsageState &rhs) const {
         return ((_diskState == rhs._diskState) &&
                 (_memoryState == rhs._memoryState) &&
+                (_transient_disk_usage == rhs._transient_disk_usage) &&
                 (_transient_memory_usage == rhs._transient_memory_usage));
     }
     bool operator!=(const DiskMemUsageState &rhs) const {
@@ -36,6 +40,7 @@ public:
     }
     const ResourceUsageState &diskState() const { return _diskState; }
     const ResourceUsageState &memoryState() const { return _memoryState; }
+    double transient_disk_usage() const { return _transient_disk_usage; }
     double transient_memory_usage() const { return _transient_memory_usage; }
     bool aboveDiskLimit(double resourceLimitFactor) const { return diskState().aboveLimit(resourceLimitFactor); }
     bool aboveMemoryLimit(double resourceLimitFactor) const { return memoryState().aboveLimit(resourceLimitFactor); }
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
index b9903d626aa..f052d663ba6 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp
@@ -97,14 +97,11 @@ public:
     DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept
         : _doc_db(doc_db)
     {}
-    size_t get_transient_memory_usage() const override {
-        return _doc_db.getReadySubDB()->getSearchableStats().memoryUsage().allocatedBytes();
-    }
-    size_t get_transient_disk_usage() const override {
-        // We estimate the transient disk usage for the next disk index fusion
-        // as the size of the largest disk index.
-        // TODO: Change this to actually measure the size of the fusion disk index(es).
-        return _doc_db.getReadySubDB()->getSearchableStats().max_component_size_on_disk();
+    TransientResourceUsage get_transient_resource_usage() const override {
+        // Transient disk usage is measured as the total disk usage of all current fusion indexes.
+        // Transient memory usage is measured as the total memory usage of all memory indexes.
+        auto stats = _doc_db.getReadySubDB()->getSearchableStats();
+        return {stats.fusion_size_on_disk(), stats.memoryUsage().allocatedBytes()};
     }
 };
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 7c52d3d96ff..b128fe16e5e 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -758,7 +758,7 @@ Proton::updateMetrics(const metrics::MetricLockGuard &)
         metrics.resourceUsage.memory.set(dm_metrics.get_memory_usage());
         metrics.resourceUsage.memoryUtilization.set(dm_metrics.get_memory_utilization());
         metrics.resourceUsage.transient_memory.set(dm_metrics.get_transient_memory_usage());
-        metrics.resourceUsage.transient_disk.set(usageFilter.get_relative_transient_disk_usage());
+        metrics.resourceUsage.transient_disk.set(dm_metrics.get_transient_disk_usage());
         metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount());
         metrics.resourceUsage.openFileDescriptors.set(FastOS_File::count_open_files());
         metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0));
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
index 4b5e97a827b..3750c5afb5f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
@@ -44,16 +44,16 @@ ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool
         disk.setDouble("usage", usageState.diskState().usage());
         disk.setDouble("limit", usageState.diskState().limit());
         disk.setDouble("utilization", usageState.diskState().utilization());
+        disk.setDouble("transient", usageState.transient_disk_usage());
         convertDiskStatsToSlime(_usage_filter.getHwInfo(), _usage_filter.getDiskUsedSize(), disk.setObject("stats"));
 
         Cursor &memory = object.setObject("memory");
         memory.setDouble("usage", usageState.memoryState().usage());
         memory.setDouble("limit", usageState.memoryState().limit());
         memory.setDouble("utilization", usageState.memoryState().utilization());
+        memory.setDouble("transient", usageState.transient_memory_usage());
         memory.setLong("physicalMemory", _usage_filter.getHwInfo().memory().sizeBytes());
         convertMemoryStatsToSlime(_usage_filter.getMemoryStats(), memory.setObject("stats"));
-        size_t transient_memory = _usage_filter.get_transient_memory_usage();
-        memory.setLong("transient", transient_memory);
 
         Cursor &address_space = object.setObject("attribute_address_space");
         address_space.setDouble("usage", attr_usage.get_usage());