Add metric for node availbility ratio

Separate node type dimensions are used for distributors and
storage nodes

3 files changed, 62 insertions, 6 deletions

diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java
index 199e9a3169b..bc66980db75 100644
--- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java
+++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java
@@ -23,6 +23,15 @@ public class MetricUpdater {
         return metricReporter.createContext(dimensions);
     }
 
+    private static int nodesInAvailableState(Map<State, Integer> nodeCounts) {
+        return nodeCounts.getOrDefault(State.INITIALIZING, 0)
+                + nodeCounts.getOrDefault(State.RETIRED, 0)
+                + nodeCounts.getOrDefault(State.UP, 0)
+                // Even though technically not true, we here treat Maintenance as an available state to
+                // avoid triggering false alerts when a node is taken down transiently in an orchestrated manner.
+                + nodeCounts.getOrDefault(State.MAINTENANCE, 0);
+    }
+
     public void updateClusterStateMetrics(ContentCluster cluster, ClusterState state) {
         Map<String, String> dimensions = new HashMap<>();
         dimensions.put("cluster", cluster.getName());
@@ -42,6 +51,10 @@ public class MetricUpdater {
                 String name = s.toString().toLowerCase() + ".count";
                 metricReporter.set(name, nodeCounts.get(s), context);
             }
+
+            final int availableNodes = nodesInAvailableState(nodeCounts);
+            final int totalNodes = Math.max(cluster.getConfiguredNodes().size(), 1); // Assumes 1-1 between distributor and storage
+            metricReporter.set("available-nodes.ratio", (double)availableNodes / totalNodes, context);
         }
         dimensions.remove("node-type");
         MetricReporter.Context context = createContext(dimensions);
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java
index adc804c805d..456395d4677 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java
@@ -10,8 +10,10 @@ import java.util.Map;
 
 import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.hasMetricContext;
 import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.withDimension;
+import static org.hamcrest.Matchers.closeTo;
 import static org.mockito.Matchers.any;
 import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.doubleThat;
 import static org.mockito.Matchers.eq;
 import static org.mockito.Mockito.mock;
 import static org.mockito.Mockito.verify;
@@ -22,11 +24,18 @@ public class MetricReporterTest {
     private static class Fixture {
         final MetricReporter mockReporter = mock(MetricReporter.class);
         final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0);
-        final ClusterFixture clusterFixture = ClusterFixture.forFlatCluster(10);
+        final ClusterFixture clusterFixture;
 
         Fixture() {
-            when(mockReporter.createContext(any())).then(invocation ->
-                    new HasMetricContext.MockContext((Map<String, ?>)invocation.getArguments()[0]));
+            this(10);
+        }
+
+        Fixture(int nodes) {
+            clusterFixture = ClusterFixture.forFlatCluster(nodes);
+            when(mockReporter.createContext(any())).then(invocation -> {
+                @SuppressWarnings("unchecked") Map<String, ?> arg = (Map<String, ?>)invocation.getArguments()[0];
+                return new HasMetricContext.MockContext(arg);
+            });
         }
     }
 
@@ -57,4 +66,38 @@ public class MetricReporterTest {
                 argThat(hasMetricContext(withNodeTypeDimension("storage"))));
     }
 
+    private void doTestRatiosInState(String clusterState, double distributorRatio, double storageRatio) {
+        Fixture f = new Fixture();
+        f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString(clusterState));
+
+        verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"),
+                doubleThat(closeTo(distributorRatio, 0.0001)),
+                argThat(hasMetricContext(withNodeTypeDimension("distributor"))));
+
+        verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"),
+                doubleThat(closeTo(storageRatio, 0.0001)),
+                argThat(hasMetricContext(withNodeTypeDimension("storage"))));
+    }
+
+    @Test
+    public void metrics_are_emitted_for_partial_node_availability_ratio() {
+        // Only Up, Init, Retired and Maintenance are counted as available states
+        doTestRatiosInState("distributor:10 .1.s:d storage:9 .1.s:d .2.s:m .4.s:r .5.s:i .6.s:s", 0.9, 0.7);
+    }
+
+    @Test
+    public void metrics_are_emitted_for_full_node_availability_ratio() {
+        doTestRatiosInState("distributor:10 storage:10", 1.0, 1.0);
+    }
+
+    @Test
+    public void metrics_are_emitted_for_zero_node_availability_ratio() {
+        doTestRatiosInState("cluster:d", 0.0, 0.0);
+    }
+
+    @Test
+    public void maintenance_mode_is_counted_as_available() {
+        doTestRatiosInState("distributor:10 storage:10 .0.s:m", 1.0, 1.0);
+    }
+
 }
diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java
index dcc9055f0d1..8c45b39bd81 100644
--- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java
+++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java
@@ -13,9 +13,9 @@ import java.util.stream.Stream;
 
 public class HasMetricContext extends ArgumentMatcher<MetricReporter.Context> {
 
-    private final Map<String, String> dimensions;
+    private final Map<String, ?> dimensions;
 
-    public HasMetricContext(Map<String, String> dimensions) {
+    private HasMetricContext(Map<String, String> dimensions) {
         this.dimensions = new TreeMap<>(dimensions);
     }
 
@@ -41,7 +41,7 @@ public class HasMetricContext extends ArgumentMatcher<MetricReporter.Context> {
         final String name;
         final String value;
 
-        public Dimension(String name, String value) {
+        private Dimension(String name, String value) {
             this.name = name;
             this.value = value;
         }