diff options
Diffstat (limited to 'clustercontroller-core')
3 files changed, 62 insertions, 6 deletions
diff --git a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java index 199e9a3169b..bc66980db75 100644 --- a/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java +++ b/clustercontroller-core/src/main/java/com/yahoo/vespa/clustercontroller/core/MetricUpdater.java @@ -23,6 +23,15 @@ public class MetricUpdater { return metricReporter.createContext(dimensions); } + private static int nodesInAvailableState(Map<State, Integer> nodeCounts) { + return nodeCounts.getOrDefault(State.INITIALIZING, 0) + + nodeCounts.getOrDefault(State.RETIRED, 0) + + nodeCounts.getOrDefault(State.UP, 0) + // Even though technically not true, we here treat Maintenance as an available state to + // avoid triggering false alerts when a node is taken down transiently in an orchestrated manner. + + nodeCounts.getOrDefault(State.MAINTENANCE, 0); + } + public void updateClusterStateMetrics(ContentCluster cluster, ClusterState state) { Map<String, String> dimensions = new HashMap<>(); dimensions.put("cluster", cluster.getName()); @@ -42,6 +51,10 @@ public class MetricUpdater { String name = s.toString().toLowerCase() + ".count"; metricReporter.set(name, nodeCounts.get(s), context); } + + final int availableNodes = nodesInAvailableState(nodeCounts); + final int totalNodes = Math.max(cluster.getConfiguredNodes().size(), 1); // Assumes 1-1 between distributor and storage + metricReporter.set("available-nodes.ratio", (double)availableNodes / totalNodes, context); } dimensions.remove("node-type"); MetricReporter.Context context = createContext(dimensions); diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java index adc804c805d..456395d4677 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/MetricReporterTest.java @@ -10,8 +10,10 @@ import java.util.Map; import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.hasMetricContext; import static com.yahoo.vespa.clustercontroller.core.matchers.HasMetricContext.withDimension; +import static org.hamcrest.Matchers.closeTo; import static org.mockito.Matchers.any; import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.doubleThat; import static org.mockito.Matchers.eq; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -22,11 +24,18 @@ public class MetricReporterTest { private static class Fixture { final MetricReporter mockReporter = mock(MetricReporter.class); final MetricUpdater metricUpdater = new MetricUpdater(mockReporter, 0); - final ClusterFixture clusterFixture = ClusterFixture.forFlatCluster(10); + final ClusterFixture clusterFixture; Fixture() { - when(mockReporter.createContext(any())).then(invocation -> - new HasMetricContext.MockContext((Map<String, ?>)invocation.getArguments()[0])); + this(10); + } + + Fixture(int nodes) { + clusterFixture = ClusterFixture.forFlatCluster(nodes); + when(mockReporter.createContext(any())).then(invocation -> { + @SuppressWarnings("unchecked") Map<String, ?> arg = (Map<String, ?>)invocation.getArguments()[0]; + return new HasMetricContext.MockContext(arg); + }); } } @@ -57,4 +66,38 @@ public class MetricReporterTest { argThat(hasMetricContext(withNodeTypeDimension("storage")))); } + private void doTestRatiosInState(String clusterState, double distributorRatio, double storageRatio) { + Fixture f = new Fixture(); + f.metricUpdater.updateClusterStateMetrics(f.clusterFixture.cluster(), ClusterState.stateFromString(clusterState)); + + verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"), + doubleThat(closeTo(distributorRatio, 0.0001)), + argThat(hasMetricContext(withNodeTypeDimension("distributor")))); + + verify(f.mockReporter).set(eq("cluster-controller.available-nodes.ratio"), + doubleThat(closeTo(storageRatio, 0.0001)), + argThat(hasMetricContext(withNodeTypeDimension("storage")))); + } + + @Test + public void metrics_are_emitted_for_partial_node_availability_ratio() { + // Only Up, Init, Retired and Maintenance are counted as available states + doTestRatiosInState("distributor:10 .1.s:d storage:9 .1.s:d .2.s:m .4.s:r .5.s:i .6.s:s", 0.9, 0.7); + } + + @Test + public void metrics_are_emitted_for_full_node_availability_ratio() { + doTestRatiosInState("distributor:10 storage:10", 1.0, 1.0); + } + + @Test + public void metrics_are_emitted_for_zero_node_availability_ratio() { + doTestRatiosInState("cluster:d", 0.0, 0.0); + } + + @Test + public void maintenance_mode_is_counted_as_available() { + doTestRatiosInState("distributor:10 storage:10 .0.s:m", 1.0, 1.0); + } + } diff --git a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java index dcc9055f0d1..8c45b39bd81 100644 --- a/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java +++ b/clustercontroller-core/src/test/java/com/yahoo/vespa/clustercontroller/core/matchers/HasMetricContext.java @@ -13,9 +13,9 @@ import java.util.stream.Stream; public class HasMetricContext extends ArgumentMatcher<MetricReporter.Context> { - private final Map<String, String> dimensions; + private final Map<String, ?> dimensions; - public HasMetricContext(Map<String, String> dimensions) { + private HasMetricContext(Map<String, String> dimensions) { this.dimensions = new TreeMap<>(dimensions); } @@ -41,7 +41,7 @@ public class HasMetricContext extends ArgumentMatcher<MetricReporter.Context> { final String name; final String value; - public Dimension(String name, String value) { + private Dimension(String name, String value) { this.name = name; this.value = value; } |