Add metrics on how far away from disk and memory resource limits proton is.

8 files changed, 59 insertions, 46 deletions

diff --git a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
index 4dd8c43d9b2..fb0c9b1f3a9 100644
--- a/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
+++ b/searchcore/src/tests/proton/server/disk_mem_usage_filter/disk_mem_usage_filter_test.cpp
@@ -3,14 +3,12 @@
 #include <vespa/vespalib/testkit/testapp.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
 #include <vespa/searchcore/proton/server/disk_mem_usage_filter.h>
+#include <vespa/searchcore/proton/server/resource_usage_state.h>
 
-using proton::DiskMemUsageFilter;
-using proton::HwInfo;
+using namespace proton;
 
 namespace fs = std::experimental::filesystem;
 
-namespace {
-
 struct Fixture
 {
     DiskMemUsageFilter _filter;
@@ -18,13 +16,13 @@ struct Fixture
     using Config = DiskMemUsageFilter::Config;
 
     Fixture()
-        : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(64 * 1024 * 1024), HwInfo::Cpu(0)))
+        : _filter(HwInfo(HwInfo::Disk(100, false, false), HwInfo::Memory(1000), HwInfo::Cpu(0)))
     {
-        _filter.setDiskUsedSize(0);
-        _filter.setMemoryStats(vespalib::ProcessMemoryStats(10000000,
-                                                            10000001,
-                                                            10000002,
-                                                            10000003,
+        _filter.setDiskUsedSize(20);
+        _filter.setMemoryStats(vespalib::ProcessMemoryStats(297,
+                                                            298,
+                                                            299,
+                                                            300,
                                                             42));
     }
 
@@ -48,16 +46,14 @@ struct Fixture
 
     void triggerMemoryLimit()
     {
-        _filter.setMemoryStats(vespalib::ProcessMemoryStats(58720259,
-                                                            58720258,
-                                                            58720257,
-                                                            58720256,
+        _filter.setMemoryStats(vespalib::ProcessMemoryStats(897,
+                                                            898,
+                                                            899,
+                                                            900,
                                                             43));
     }
 };
 
-}
-
 TEST_F("Check that default filter allows write", Fixture)
 {
     f.testWrite("");
@@ -70,28 +66,40 @@ TEST_F("Check that stats are wired through", Fixture)
     EXPECT_EQUAL(43u, f._filter.getMemoryStats().getMappingsCount());
 }
 
+void
+assertResourceUsage(double usage, double limit, double utilization, const ResourceUsageState &state)
+{
+    EXPECT_EQUAL(usage, state.usage());
+    EXPECT_EQUAL(limit, state.limit());
+    EXPECT_EQUAL(utilization, state.utilization());
+}
+
 TEST_F("Check that disk limit can be reached", Fixture)
 {
     f._filter.setConfig(Fixture::Config(1.0, 0.8));
+    TEST_DO(assertResourceUsage(0.2, 0.8, 0.25, f._filter.usageState().diskState()));
     f.triggerDiskLimit();
     f.testWrite("diskLimitReached: { "
                 "action: \"add more content nodes\", "
                 "reason: \"disk used (0.9) > disk limit (0.8)\", "
                 "stats: { "
                 "capacity: 100, used: 90, diskUsed: 0.9, diskLimit: 0.8}}");
+    TEST_DO(assertResourceUsage(0.9, 0.8, 1.125, f._filter.usageState().diskState()));
 }
 
 TEST_F("Check that memory limit can be reached", Fixture)
 {
     f._filter.setConfig(Fixture::Config(0.8, 1.0));
+    TEST_DO(assertResourceUsage(0.3, 0.8, 0.375, f._filter.usageState().memoryState()));
     f.triggerMemoryLimit();
     f.testWrite("memoryLimitReached: { "
                 "action: \"add more content nodes\", "
-                "reason: \"memory used (0.875) > memory limit (0.8)\", "
+                "reason: \"memory used (0.9) > memory limit (0.8)\", "
                 "stats: { "
-                "mapped: { virt: 58720259, rss: 58720258}, "
-                "anonymous: { virt: 58720257, rss: 58720256}, "
-                "physicalMemory: 67108864, memoryUsed: 0.875, memoryLimit: 0.8}}");
+                "mapped: { virt: 897, rss: 898}, "
+                "anonymous: { virt: 899, rss: 900}, "
+                "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}");
+    TEST_DO(assertResourceUsage(0.9, 0.8, 1.125, f._filter.usageState().memoryState()));
 }
 
 TEST_F("Check that both disk limit and memory limit can be reached", Fixture)
@@ -101,11 +109,11 @@ TEST_F("Check that both disk limit and memory limit can be reached", Fixture)
     f.triggerDiskLimit();
     f.testWrite("memoryLimitReached: { "
                 "action: \"add more content nodes\", "
-                "reason: \"memory used (0.875) > memory limit (0.8)\", "
+                "reason: \"memory used (0.9) > memory limit (0.8)\", "
                 "stats: { "
-                "mapped: { virt: 58720259, rss: 58720258}, "
-                "anonymous: { virt: 58720257, rss: 58720256}, "
-                "physicalMemory: 67108864, memoryUsed: 0.875, memoryLimit: 0.8}}, "
+                "mapped: { virt: 897, rss: 898}, "
+                "anonymous: { virt: 899, rss: 900}, "
+                "physicalMemory: 1000, memoryUsed: 0.9, memoryLimit: 0.8}}, "
                 "diskLimitReached: { "
                 "action: \"add more content nodes\", "
                 "reason: \"disk used (0.9) > disk limit (0.8)\", "
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp
index a699cf491e2..d944287243a 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp
+++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.cpp
@@ -7,7 +7,9 @@ namespace proton {
 ResourceUsageMetrics::ResourceUsageMetrics(metrics::MetricSet *parent)
     : MetricSet("resource_usage", "", "Usage metrics for various resources in this search engine", parent),
       disk("disk", "", "The relative amount of disk space used on this machine (value in the range [0, 1])", this),
+      diskUtilization("disk_utilization", "", "The relative amount of disk used compared to the disk resource limit", this),
       memory("memory", "", "The relative amount of memory used by this process (value in the range [0, 1])", this),
+      memoryUtilization("memory_utilization", "", "The relative amount of memory used compared to the memory resource limit", this),
       memoryMappings("memory_mappings", "", "The number of mapped memory areas", this),
       openFileDescriptors("open_file_descriptors", "", "The number of open files", this),
       feedingBlocked("feeding_blocked", "", "Whether feeding is blocked due to resource limits being reached (value is either 0 or 1)", this)
diff --git a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h
index 698ba8029bd..497a0fe2ba0 100644
--- a/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h
+++ b/searchcore/src/vespa/searchcore/proton/metrics/resource_usage_metrics.h
@@ -12,7 +12,9 @@ namespace proton {
 struct ResourceUsageMetrics : metrics::MetricSet
 {
     metrics::DoubleValueMetric disk;
+    metrics::DoubleValueMetric diskUtilization;
     metrics::DoubleValueMetric memory;
+    metrics::DoubleValueMetric memoryUtilization;
     metrics::LongValueMetric memoryMappings;
     metrics::LongValueMetric openFileDescriptors;
     metrics::LongValueMetric feedingBlocked;
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
index 32b2b6aaba6..d53adf19f76 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.cpp
@@ -209,18 +209,11 @@ DiskMemUsageFilter::getConfig() const
     return _config;
 }
 
-double
-DiskMemUsageFilter::getMemoryUsedRatio() const
-{
-    Guard guard(_lock);
-    return getMemoryUsedRatio(guard);
-}
-
-double
-DiskMemUsageFilter::getDiskUsedRatio() const
+DiskMemUsageState
+DiskMemUsageFilter::usageState() const
 {
     Guard guard(_lock);
-    return getDiskUsedRatio(guard);
+    return _dmstate;
 }
 
 bool
diff --git a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
index 4906348a54d..982e3bba6fd 100644
--- a/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
+++ b/searchcore/src/vespa/searchcore/proton/server/disk_mem_usage_filter.h
@@ -64,8 +64,7 @@ public:
     uint64_t getDiskUsedSize() const;
     Config getConfig() const;
     const HwInfo &getHwInfo() const { return _hwInfo; }
-    double getMemoryUsedRatio() const;
-    double getDiskUsedRatio() const;
+    DiskMemUsageState usageState() const;
     bool acceptWriteOperation() const override;
     State getAcceptState() const override;
     void addDiskMemUsageListener(IDiskMemUsageListener *listener) override;
diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
index 6de5410470f..ecdcf1c9c0f 100644
--- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp
@@ -704,8 +704,11 @@ Proton::updateMetrics(const vespalib::MonitorGuard &)
         ContentProtonMetrics &metrics = _metricsEngine->root();
         metrics.transactionLog.update(_tls->getTransLogServer()->getDomainStats());
         const DiskMemUsageFilter &usageFilter = _diskMemUsageSampler->writeFilter();
-        metrics.resourceUsage.disk.set(usageFilter.getDiskUsedRatio());
-        metrics.resourceUsage.memory.set(usageFilter.getMemoryUsedRatio());
+        DiskMemUsageState usageState = usageFilter.usageState();
+        metrics.resourceUsage.disk.set(usageState.diskState().usage());
+        metrics.resourceUsage.diskUtilization.set(usageState.diskState().utilization());
+        metrics.resourceUsage.memory.set(usageState.memoryState().usage());
+        metrics.resourceUsage.memoryUtilization.set(usageState.memoryState().utilization());
         metrics.resourceUsage.memoryMappings.set(usageFilter.getMemoryStats().getMappingsCount());
         metrics.resourceUsage.openFileDescriptors.set(countOpenFiles());
         metrics.resourceUsage.feedingBlocked.set((usageFilter.acceptWriteOperation() ? 0.0 : 1.0));
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
index 33b37649ed2..6f1fd8b90e0 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_explorer.cpp
@@ -33,21 +33,23 @@ void
 ResourceUsageExplorer::get_state(const vespalib::slime::Inserter &inserter, bool full) const
 {
     Cursor &object = inserter.insertObject();
+    DiskMemUsageState usageState = _usageFilter.usageState();
     if (full) {
-        DiskMemUsageFilter::Config config = _usageFilter.getConfig();
         Cursor &disk = object.setObject("disk");
-        disk.setDouble("usedRatio", _usageFilter.getDiskUsedRatio());
-        disk.setDouble("usedLimit", config._diskLimit);
+        disk.setDouble("usage", usageState.diskState().usage());
+        disk.setDouble("limit", usageState.diskState().limit());
+        disk.setDouble("utilization", usageState.diskState().utilization());
         convertDiskStatsToSlime(_usageFilter.getHwInfo(), _usageFilter.getDiskUsedSize(), disk.setObject("stats"));
 
         Cursor &memory = object.setObject("memory");
-        memory.setDouble("usedRatio", _usageFilter.getMemoryUsedRatio());
-        memory.setDouble("usedLimit", config._memoryLimit);
+        memory.setDouble("usage", usageState.memoryState().usage());
+        memory.setDouble("limit", usageState.memoryState().limit());
+        memory.setDouble("utilization", usageState.memoryState().utilization());
         memory.setLong("physicalMemory", _usageFilter.getHwInfo().memory().sizeBytes());
         convertMemoryStatsToSlime(_usageFilter.getMemoryStats(), memory.setObject("stats"));
     } else {
-        object.setDouble("disk", _usageFilter.getDiskUsedRatio());
-        object.setDouble("memory", _usageFilter.getMemoryUsedRatio());
+        object.setDouble("disk", usageState.diskState().usage());
+        object.setDouble("memory", usageState.memoryState().usage());
     }
 }
 
diff --git a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h
index 7ac127c6b25..29f9959aea9 100644
--- a/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h
+++ b/searchcore/src/vespa/searchcore/proton/server/resource_usage_state.h
@@ -5,7 +5,10 @@
 namespace proton {
 
 /**
- * Class representing the state of an resource (e.g. disk or memory) with its limit and current usage.
+ * Class representing the state of an resource (e.g. disk or memory) with its limit and current usage:
+ *   - usage: How much of this resource is currently used (number between 0 and 1).
+ *   - limit: How much of this resource is allowed to use (number between 0 and 1).
+ *   - utilization: How much of the allowed part of this resource is used (usage/limit).
  */
 class ResourceUsageState
 {
@@ -33,6 +36,7 @@ public:
     }
     double limit() const { return _limit; }
     double usage() const { return _usage; }
+    double utilization() const { return _usage/_limit; }
     bool aboveLimit() const {
         return aboveLimit(1.0);
     }