Let distributor metric decide cluster stability

2 files changed, 40 insertions, 3 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
index 01bcabeb11b..82f96375d79 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsResponse.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.provision.autoscale;
 
 import ai.vespa.metrics.ContainerMetrics;
+import ai.vespa.metrics.DistributorMetrics;
 import ai.vespa.metrics.HostedNodeAdminMetrics;
 import ai.vespa.metrics.SearchNodeMetrics;
 import ai.vespa.metrics.StorageMetrics;
@@ -78,7 +79,7 @@ public class MetricsResponse {
                                                                              Metric.disk.from(nodeValues)),
                                                                     (long)Metric.generation.from(nodeValues),
                                                                     Metric.inService.from(nodeValues) > 0,
-                                                                    clusterIsStable(node.get(), applicationNodes),
+                                                                    clusterIsStable(node.get(), applicationNodes, nodeValues),
                                                                     Metric.queryRate.from(nodeValues))));
 
         var cluster = node.get().allocation().get().membership().cluster().id();
@@ -107,7 +108,10 @@ public class MetricsResponse {
         item.field("values").traverse((ObjectTraverser)(name, value) -> values.put(name, value.asDouble()));
     }
 
-    private boolean clusterIsStable(Node node, NodeList applicationNodes) {
+    private boolean clusterIsStable(Node node, NodeList applicationNodes, ListMap<String, Double> nodeValues) {
+        if (Metric.redistributing.from(nodeValues) > 0) {
+            return false;
+        }
         ClusterSpec cluster = node.allocation().get().membership().cluster();
         return applicationNodes.cluster(cluster.id()).retired().isEmpty();
     }
@@ -207,6 +211,20 @@ public class MetricsResponse {
             }
 
         },
+        redistributing { // whether data redistribution is ongoing
+
+            @Override
+            public List<String> metricResponseNames() {
+                return List.of(DistributorMetrics.VDS_IDEALSTATE_MERGE_BUCKET_PENDING.last());
+            }
+
+            @Override
+            double computeFinal(ListMap<String, Double> values) {
+                // Really a bool. True if any node is merging buckets.
+                return values.values().stream().flatMap(List::stream).anyMatch(v -> v > 0) ? 1 : 0;
+            }
+
+        },
         queryRate { // queries per second
 
             @Override
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java
index 127f3525cf5..4ec4ecd6d84 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/MetricsV2MetricsFetcherTest.java
@@ -99,6 +99,7 @@ public class MetricsV2MetricsFetcherTest {
             assertEquals("host-3.yahoo.com", values.get(0).getFirst());
             assertEquals(0.13, values.get(0).getSecond().load().cpu(), delta);
             assertEquals(0.9375, values.get(0).getSecond().load().memory(), delta);
+            assertFalse("Unstable because buckets are being merged", values.get(0).getSecond().stable());
         }
 
     }
@@ -265,7 +266,25 @@ public class MetricsV2MetricsFetcherTest {
                                 }
                               }
                             ]
-                          }
+                          },
+                          "services": [
+                            {
+                              "name": "distributor",
+                              "timestamp": 1234,
+                              "status": {
+                                "code": "up"
+                              },
+                              "metrics": [
+                                {
+                                  "values": {
+                                    "vds.idealstate.merge_bucket.pending.last": 42.2
+                                  },
+                                  "dimensions": {
+                                  }
+                                }
+                              ]
+                            }
+                          ]
                         }
                       ]
                     }