Expose resource usage metrics for disk and memory limits for feed blocked.

5 files changed, 41 insertions, 5 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java
index 53e62ab3e10..6a80772904f 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java
@@ -64,6 +64,8 @@ public class MetricUpdater {
         metricReporter.set("resource_usage.max_disk_utilization", resourceUsage.getMaxDiskUtilization(), context);
         metricReporter.set("resource_usage.max_memory_utilization", resourceUsage.getMaxMemoryUtilization(), context);
         metricReporter.set("resource_usage.nodes_above_limit", resourceUsage.getNodesAboveLimit(), context);
+        metricReporter.set("resource_usage.disk_limit", resourceUsage.getDiskLimit(), context);
+        metricReporter.set("resource_usage.memory_limit", resourceUsage.getMemoryLimit(), context);
     }
 
     public void updateMasterElectionMetrics(Map<Integer, Integer> data) {
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStats.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStats.java
index a2b3c3a0415..53d801e609d 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStats.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStats.java
@@ -22,6 +22,9 @@ public class ResourceUsageStats {
     // When this is above zero feed is blocked in the cluster.
     private final int nodesAboveLimit;
 
+    private final double diskLimit;
+    private final double memoryLimit;
+
     private static final String diskResource = "disk";
     private static final String memoryResource = "memory";
 
@@ -29,14 +32,20 @@ public class ResourceUsageStats {
         this.maxDiskUtilization = 0.0;
         this.maxMemoryUtilization = 0.0;
         this.nodesAboveLimit = 0;
+        this.diskLimit = 0.0;
+        this.memoryLimit = 0.0;
     }
 
     public ResourceUsageStats(double maxDiskUtilization,
                               double maxMemoryUtilization,
-                              int nodesAboveLimit) {
+                              int nodesAboveLimit,
+                              double diskLimit,
+                              double memoryLimit) {
         this.maxDiskUtilization = maxDiskUtilization;
         this.maxMemoryUtilization = maxMemoryUtilization;
         this.nodesAboveLimit = nodesAboveLimit;
+        this.diskLimit = diskLimit;
+        this.memoryLimit = memoryLimit;
     }
 
     public double getMaxDiskUtilization() {
@@ -51,6 +60,14 @@ public class ResourceUsageStats {
         return nodesAboveLimit;
     }
 
+    public double getDiskLimit() {
+        return diskLimit;
+    }
+
+    public double getMemoryLimit() {
+        return memoryLimit;
+    }
+
     public static ResourceUsageStats calculateFrom(Collection<NodeInfo> nodeInfos,
                                                    Map<String, Double> feedBlockLimits,
                                                    Optional<ClusterStateBundle.FeedBlock> feedBlock) {
@@ -63,9 +80,12 @@ public class ResourceUsageStats {
                 maxMemoryUsage = Double.max(maxMemoryUsage, resourceUsageOf(memoryResource, node));
             }
         }
-        return new ResourceUsageStats(maxDiskUsage / limitOf(diskResource, feedBlockLimits),
-                maxMemoryUsage / limitOf(memoryResource, feedBlockLimits),
-                calculateNodesAboveLimit(feedBlock));
+        double diskLimit = limitOf(diskResource, feedBlockLimits);
+        double memoryLimit = limitOf(memoryResource, feedBlockLimits);
+        return new ResourceUsageStats(maxDiskUsage / diskLimit,
+                maxMemoryUsage / memoryLimit,
+                calculateNodesAboveLimit(feedBlock),
+                diskLimit, memoryLimit);
     }
 
     private static double resourceUsageOf(String type, ContentNode node) {
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java
index 19b73e4194f..3e44995dc78 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java
@@ -113,7 +113,7 @@ public class MetricReporterTest {
         Fixture f = new Fixture();
         f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(),
                 ClusterState.stateFromString("distributor:10 storage:10"),
-                new ResourceUsageStats(0.5, 0.6, 5));
+                new ResourceUsageStats(0.5, 0.6, 5, 0.7, 0.8));
 
         verify(f.mockReporter).set(eq("cluster-controller.resource_usage.max_disk_utilization"),
                 doubleThat(closeTo(0.5, 0.0001)),
@@ -126,6 +126,14 @@ public class MetricReporterTest {
         verify(f.mockReporter).set(eq("cluster-controller.resource_usage.nodes_above_limit"),
                 eq(5),
                 argThat(hasMetricContext(withClusterDimension())));
+
+        verify(f.mockReporter).set(eq("cluster-controller.resource_usage.disk_limit"),
+                doubleThat(closeTo(0.7, 0.0001)),
+                argThat(hasMetricContext(withClusterDimension())));
+
+        verify(f.mockReporter).set(eq("cluster-controller.resource_usage.memory_limit"),
+                doubleThat(closeTo(0.8, 0.0001)),
+                argThat(hasMetricContext(withClusterDimension())));
     }
 
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java
index 9eeb36265e0..6dbac772f16 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/ResourceUsageStatsTest.java
@@ -29,6 +29,8 @@ public class ResourceUsageStatsTest {
                 Optional.empty());
         assertEquals(0.4 / 0.8, stats.getMaxDiskUtilization(), DELTA);
         assertEquals(0.6 / 0.9, stats.getMaxMemoryUtilization(), DELTA);
+        assertEquals(0.8, stats.getDiskLimit(), DELTA);
+        assertEquals(0.9, stats.getMemoryLimit(), DELTA);
     }
 
     @Test
@@ -39,6 +41,8 @@ public class ResourceUsageStatsTest {
                 Optional.empty());
         assertEquals(0.0, stats.getMaxDiskUtilization(), DELTA);
         assertEquals(0.0, stats.getMaxMemoryUtilization(), DELTA);
+        assertEquals(0.8, stats.getDiskLimit(), DELTA);
+        assertEquals(0.9, stats.getMemoryLimit(), DELTA);
     }
 
     @Test
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
index 97c59675b63..869738223d4 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/admin/monitoring/VespaMetricSet.java
@@ -247,6 +247,8 @@ public class VespaMetricSet {
         metrics.add(new Metric("cluster-controller.resource_usage.nodes_above_limit.last"));
         metrics.add(new Metric("cluster-controller.resource_usage.max_memory_utilization.last"));
         metrics.add(new Metric("cluster-controller.resource_usage.max_disk_utilization.last"));
+        metrics.add(new Metric("cluster-controller.resource_usage.disk_limit.last"));
+        metrics.add(new Metric("cluster-controller.resource_usage.memory_limit.last"));
 
         metrics.add(new Metric("reindexing.progress.last"));