diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-12-06 11:35:35 +0100 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-12-06 11:35:35 +0100 |
commit | 1502c72a9c011e6ff036120ae7dacb94fc3b3a47 (patch) | |
tree | 21e00d4a79bb516e49de3c8b0a6c3eb430b6abef | |
parent | 3ce3357259f5aad649b5894139db088780b0a39b (diff) |
Let distributor metric decide cluster stability
2 files changed, 40 insertions, 3 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java index 01bcabeb11b..82f96375d79 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.provision.autoscale; import ai.vespa.metrics.ContainerMetrics; +import ai.vespa.metrics.DistributorMetrics; import ai.vespa.metrics.HostedNodeAdminMetrics; import ai.vespa.metrics.SearchNodeMetrics; import ai.vespa.metrics.StorageMetrics; @@ -78,7 +79,7 @@ public class MetricsResponse { Metric.disk.from(nodeValues)), (long)Metric.generation.from(nodeValues), Metric.inService.from(nodeValues) > 0, - clusterIsStable(node.get(), applicationNodes), + clusterIsStable(node.get(), applicationNodes, nodeValues), Metric.queryRate.from(nodeValues)))); var cluster = node.get().allocation().get().membership().cluster().id(); @@ -107,7 +108,10 @@ public class MetricsResponse { item.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble())); } - private boolean clusterIsStable(Node node, NodeList applicationNodes) { + private boolean clusterIsStable(Node node, NodeList applicationNodes, ListMap<String, Double> nodeValues) { + if (Metric.redistributing.from(nodeValues) > 0) { + return false; + } ClusterSpec cluster = node.allocation().get().membership().cluster(); return applicationNodes.cluster(cluster.id()).retired().isEmpty(); } @@ -207,6 +211,20 @@ public class MetricsResponse { } }, + redistributing { // whether data redistribution is ongoing + + @Override + public List<String> metricResponseNames() { + return List.of(DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_PENDING.last()); + } + + @Override + double computeFinal(ListMap<String, Double> values) { + // Really a bool. True if any node is merging buckets. + return values.values().stream().flatMap(List::stream).anyMatch(v -> v > 0) ? 1 : 0; + } + + }, queryRate { // queries per second @Override diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java index 127f3525cf5..4ec4ecd6d84 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java @@ -99,6 +99,7 @@ public class MetricsV2MetricsFetcherTest { assertEquals("host-3.yahoo.com", values.get(0).getFirst()); assertEquals(0.13, values.get(0).getSecond().load().cpu(), delta); assertEquals(0.9375, values.get(0).getSecond().load().memory(), delta); + assertFalse("Unstable because buckets are being merged", values.get(0).getSecond().stable()); } } @@ -265,7 +266,25 @@ public class MetricsV2MetricsFetcherTest { } } ] - } + }, + "services": [ + { + "name": "distributor", + "timestamp": 1234, + "status": { + "code": "up" + }, + "metrics": [ + { + "values": { + "vds.idealstate.merge_bucket.pending.last": 42.2 + }, + "dimensions": { + } + } + ] + } + ] } ] } |