Merge pull request #26263 from vespa-engine/bratseth/bcp-deadline

Use bcp deadline

8 files changed, 99 insertions, 16 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java
index dc01c37f854..fe8acb0c3c0 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/ClusterInfo.java
@@ -1,6 +1,7 @@
 package com.yahoo.config.provision;
 
 import java.time.Duration;
+import java.util.Objects;
 
 /**
  * Auxiliary information about a cluster, provided by the config model to the node repo during a
@@ -22,12 +23,32 @@ public class ClusterInfo {
 
     public static ClusterInfo empty() { return empty; }
 
+    public boolean isEmpty() { return this.equals(empty); }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == this) return true;
+        if ( ! (o instanceof ClusterInfo other)) return false;
+        if ( ! other.bcpDeadline.equals(this.bcpDeadline)) return false;
+        return true;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(bcpDeadline);
+    }
+
+    @Override
+    public String toString() {
+        return "cluster info: [bcp deadline: " + bcpDeadline + "]";
+    }
+
     public static class Builder {
 
         private Duration bcpDeadline = Duration.ofMinutes(0);
 
         public Builder bcpDeadline(Duration duration) {
-            this.bcpDeadline = duration;
+            this.bcpDeadline = Objects.requireNonNull(duration);
             return this;
         }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
index ea35f1e85ff..16016815b7c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/applications/Cluster.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.applications;
 
+import com.yahoo.config.provision.ClusterInfo;
 import com.yahoo.config.provision.IntRange;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.ClusterResources;
@@ -33,6 +34,7 @@ public class Cluster {
     private final boolean required;
     private final Autoscaling suggested;
     private final Autoscaling target;
+    private final ClusterInfo clusterInfo;
     private final BcpGroupInfo bcpGroupInfo;
 
     /** The maxScalingEvents last scaling events of this, sorted by increasing time (newest last) */
@@ -46,6 +48,7 @@ public class Cluster {
                    boolean required,
                    Autoscaling suggested,
                    Autoscaling target,
+                   ClusterInfo clusterInfo,
                    BcpGroupInfo bcpGroupInfo,
                    List<ScalingEvent> scalingEvents) {
         this.id = Objects.requireNonNull(id);
@@ -60,6 +63,7 @@ public class Cluster {
             this.target = target.withResources(Optional.empty()); // Delete illegal target
         else
             this.target = target;
+        this.clusterInfo = clusterInfo;
         this.bcpGroupInfo = Objects.requireNonNull(bcpGroupInfo);
         this.scalingEvents = List.copyOf(scalingEvents);
     }
@@ -105,6 +109,8 @@ public class Cluster {
         return true;
     }
 
+    public ClusterInfo clusterInfo() { return clusterInfo; }
+
     /** Returns info about the BCP group of clusters this belongs to. */
     public BcpGroupInfo bcpGroupInfo() { return bcpGroupInfo; }
 
@@ -119,19 +125,19 @@ public class Cluster {
     public Cluster withConfiguration(boolean exclusive, Capacity capacity) {
         return new Cluster(id, exclusive,
                            capacity.minResources(), capacity.maxResources(), capacity.groupSize(), capacity.isRequired(),
-                           suggested, target, bcpGroupInfo, scalingEvents);
+                           suggested, target, capacity.clusterInfo(), bcpGroupInfo, scalingEvents);
     }
 
     public Cluster withSuggested(Autoscaling suggested) {
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     public Cluster withTarget(Autoscaling target) {
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     public Cluster with(BcpGroupInfo bcpGroupInfo) {
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     /** Add or update (based on "at" time) a scaling event */
@@ -145,7 +151,7 @@ public class Cluster {
             scalingEvents.add(scalingEvent);
 
         prune(scalingEvents);
-        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, bcpGroupInfo, scalingEvents);
+        return new Cluster(id, exclusive, min, max, groupSize, required, suggested, target, clusterInfo, bcpGroupInfo, scalingEvents);
     }
 
     @Override
@@ -177,7 +183,7 @@ public class Cluster {
     public static Cluster create(ClusterSpec.Id id, boolean exclusive, Capacity requested) {
         return new Cluster(id, exclusive,
                            requested.minResources(), requested.maxResources(), requested.groupSize(), requested.isRequired(),
-                           Autoscaling.empty(), Autoscaling.empty(), BcpGroupInfo.empty(), List.of());
+                           Autoscaling.empty(), Autoscaling.empty(), requested.clusterInfo(), BcpGroupInfo.empty(), List.of());
     }
 
     /** The predicted time it will take to rescale this cluster. */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
index 3f916f1cc59..4e1523834e4 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/autoscale/ClusterModel.java
@@ -193,23 +193,28 @@ public class ClusterModel {
 
     /**
      * Returns the ideal load across the nodes of this such that each node will be at ideal load
-     * if one of  the nodes go down.
+     * if one of the nodes go down.
      */
     public Load idealLoad() {
         var ideal = new Load(idealCpuLoad(), idealMemoryLoad(), idealDiskLoad()).divide(redundancyAdjustment());
-        if (! cluster.bcpGroupInfo().isEmpty()) {
+        if ( !cluster.bcpGroupInfo().isEmpty() && cluster.bcpGroupInfo().queryRate() > 0) {
+            // Since we have little local information, use information about query cost in other groups
+
+            Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal);
+
             // Do a weighted sum of the ideal "vote" based on local and bcp group info.
             // This avoids any discontinuities with a near-zero local query rate.
-            double localInformationWeight = cluster.bcpGroupInfo().queryRate() == 0
-                                            ? 1
-                                            : Math.min(1, averageQueryRate().orElse(0) /
-                                                          Math.min(queryRateGivingFullConfidence, cluster.bcpGroupInfo().queryRate()));
-            Load bcpGroupIdeal = adjustQueryDependentIdealLoadByBcpGroupInfo(ideal);
+            double localInformationWeight = Math.min(1, averageQueryRate().orElse(0) /
+                                                        Math.min(queryRateGivingFullConfidence, cluster.bcpGroupInfo().queryRate()));
             ideal = ideal.multiply(localInformationWeight).add(bcpGroupIdeal.multiply(1 - localInformationWeight));
         }
         return ideal;
     }
 
+    private boolean canRescaleWithinBcpDeadline() {
+        return scalingDuration().minus(cluster.clusterInfo().bcpDeadline()).isNegative();
+    }
+
     public Autoscaling.Metrics metrics() {
         return new Autoscaling.Metrics(averageQueryRate().orElse(0),
                                        growthRateHeadroom(),
@@ -230,7 +235,9 @@ public class ClusterModel {
     private Load adjustQueryDependentIdealLoadByBcpGroupInfo(Load ideal) {
         double currentClusterTotalVcpuPerGroup = nodes.not().retired().first().get().resources().vcpu() * groupSize();
 
-        double targetQueryRateToHandle = cluster.bcpGroupInfo().queryRate() * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom();
+        double targetQueryRateToHandle = ( canRescaleWithinBcpDeadline() ? averageQueryRate().orElse(0)
+                                                                         : cluster.bcpGroupInfo().queryRate() )
+                                         * cluster.bcpGroupInfo().growthRateHeadroom() * trafficShiftHeadroom();
         double neededTotalVcpPerGroup = cluster.bcpGroupInfo().cpuCostPerQuery() * targetQueryRateToHandle / groupCount() +
                                         ( 1 - queryCpuFraction()) * idealCpuLoad() *
                                         (clusterSpec.type().isContainer() ? 1 : groupSize());
@@ -325,6 +332,7 @@ public class ClusterModel {
      */
     private double trafficShiftHeadroom() {
         if ( ! zone.environment().isProduction()) return 1;
+        if (canRescaleWithinBcpDeadline()) return 1;
         double trafficShiftHeadroom;
         if (application.status().maxReadShare() == 0) // No traffic fraction data
             trafficShiftHeadroom = 2.0; // assume we currently get half of the max possible share of traffic
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
index 38675ba3758..1e378c80f90 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializer.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.persistence;
 
+import com.yahoo.config.provision.ClusterInfo;
 import com.yahoo.config.provision.IntRange;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterResources;
@@ -20,6 +21,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Load;
 
 import java.io.IOException;
 import java.io.UncheckedIOException;
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -54,6 +56,8 @@ public class ApplicationSerializer {
     private static final String groupSizeKey = "groupSize";
     private static final String requiredKey = "required";
     private static final String suggestedKey = "suggested";
+    private static final String clusterInfoKey = "clusterInfo";
+    private static final String bcpDeadlineKey = "bcpDeadline";
     private static final String bcpGroupInfoKey = "bcpGroupInfo";
     private static final String queryRateKey = "queryRateKey";
     private static final String growthRateHeadroomKey = "growthRateHeadroomKey";
@@ -134,6 +138,8 @@ public class ApplicationSerializer {
         clusterObject.setBool(requiredKey, cluster.required());
         toSlime(cluster.suggested(), clusterObject.setObject(suggestedKey));
         toSlime(cluster.target(), clusterObject.setObject(targetKey));
+        if (! cluster.clusterInfo().isEmpty())
+            toSlime(cluster.clusterInfo(), clusterObject.setObject(clusterInfoKey));
         if (! cluster.bcpGroupInfo().isEmpty())
             toSlime(cluster.bcpGroupInfo(), clusterObject.setObject(bcpGroupInfoKey));
         scalingEventsToSlime(cluster.scalingEvents(), clusterObject.setArray(scalingEventsKey));
@@ -148,6 +154,7 @@ public class ApplicationSerializer {
                            clusterObject.field(requiredKey).asBool(),
                            autoscalingFromSlime(clusterObject.field(suggestedKey)),
                            autoscalingFromSlime(clusterObject.field(targetKey)),
+                           clusterInfoFromSlime(clusterObject.field(clusterInfoKey)),
                            bcpGroupInfoFromSlime(clusterObject.field(bcpGroupInfoKey)),
                            scalingEventsFromSlime(clusterObject.field(scalingEventsKey)));
     }
@@ -225,8 +232,18 @@ public class ApplicationSerializer {
                                metricsFromSlime(autoscalingObject.field(metricsKey)));
     }
 
+    private static void toSlime(ClusterInfo clusterInfo, Cursor clusterInfoObject) {
+        clusterInfoObject.setLong(bcpDeadlineKey, clusterInfo.bcpDeadline().toMinutes());
+    }
+
+    private static ClusterInfo clusterInfoFromSlime(Inspector clusterInfoObject) {
+        if ( ! clusterInfoObject.valid()) return ClusterInfo.empty();
+        ClusterInfo.Builder builder = new ClusterInfo.Builder();
+        builder.bcpDeadline(Duration.ofMinutes(clusterInfoObject.field(bcpDeadlineKey).asLong()));
+        return builder.build();
+    }
+
     private static void toSlime(BcpGroupInfo bcpGroupInfo, Cursor bcpGroupInfoObject) {
-        if (bcpGroupInfo.isEmpty()) return;
         bcpGroupInfoObject.setDouble(queryRateKey, bcpGroupInfo.queryRate());
         bcpGroupInfoObject.setDouble(growthRateHeadroomKey, bcpGroupInfo.growthRateHeadroom());
         bcpGroupInfoObject.setDouble(cpuCostPerQueryKey, bcpGroupInfo.cpuCostPerQuery());
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java
index bd3589be9dd..704491ed44f 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/AutoscalingUsingBcpGroupInfoTest.java
@@ -1,8 +1,10 @@
 package com.yahoo.vespa.hosted.provision.autoscale;
 
 import com.yahoo.config.provision.Capacity;
+import com.yahoo.config.provision.ClusterInfo;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.IntRange;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.vespa.hosted.provision.applications.BcpGroupInfo;
 import com.yahoo.vespa.hosted.provision.provisioning.DynamicProvisioningTester;
@@ -154,6 +156,28 @@ public class AutoscalingUsingBcpGroupInfoTest {
     }
 
     @Test
+    public void test_autoscaling_with_bcp_deadline() {
+        var capacity = Capacity.from(new ClusterResources(2, 1,
+                                                          new NodeResources(1, 4, 10, 1, NodeResources.DiskSpeed.any)),
+                                     new ClusterResources(20, 1,
+                                                          new NodeResources(100, 1000, 1000, 1, NodeResources.DiskSpeed.any)),
+                                     IntRange.empty(), false, true, Optional.empty(),
+                                     new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(60)).build());
+
+        var fixture = DynamicProvisioningTester.fixture()
+                                               .capacity(capacity)
+                                               .clusterType(ClusterSpec.Type.container).awsProdSetup(true).build();
+
+        // We can rescale within deadline - do not take BCP info into account
+        fixture.tester().clock().advance(Duration.ofDays(2));
+        fixture.store(new BcpGroupInfo(100, 1.1, 0.45));
+        fixture.loader().addCpuMeasurements(0.7f, 10);
+        fixture.tester().assertResources("No need for traffic shift headroom",
+                                         2, 1, 2.0, 16.0, 40.8,
+                                         fixture.autoscale());
+    }
+
+    @Test
     public void test_autoscaling_single_content_group_with_some_local_traffic() {
         var fixture = DynamicProvisioningTester.fixture().awsProdSetup(true).build();
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
index f9dd4578f7a..48db3fade95 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/autoscale/Fixture.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Capacity;
 import com.yahoo.config.provision.Cloud;
+import com.yahoo.config.provision.ClusterInfo;
 import com.yahoo.config.provision.ClusterResources;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java
index d04fe1bdda2..c429f88cfa1 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/persistence/ApplicationSerializerTest.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.persistence;
 
+import com.yahoo.config.provision.ClusterInfo;
 import com.yahoo.config.provision.IntRange;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterResources;
@@ -15,6 +16,7 @@ import com.yahoo.vespa.hosted.provision.autoscale.Autoscaling;
 import com.yahoo.vespa.hosted.provision.autoscale.Load;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
@@ -40,6 +42,7 @@ public class ApplicationSerializerTest {
                                  true,
                                  Autoscaling.empty(),
                                  Autoscaling.empty(),
+                                 ClusterInfo.empty(),
                                  BcpGroupInfo.empty(),
                                  List.of()));
         var minResources = new NodeResources(1, 2, 3, 4);
@@ -65,6 +68,7 @@ public class ApplicationSerializerTest {
                                                  Load.zero(),
                                                  Load.one(),
                                                  Autoscaling.Metrics.zero()),
+                                 new ClusterInfo.Builder().bcpDeadline(Duration.ofMinutes(33)).build(),
                                  new BcpGroupInfo(0.1, 0.2, 0.3),
                                  List.of(new ScalingEvent(new ClusterResources(10, 5, minResources),
                                                           new ClusterResources(12, 6, minResources),
@@ -95,6 +99,7 @@ public class ApplicationSerializerTest {
             assertEquals(originalCluster.required(), serializedCluster.required());
             assertEquals(originalCluster.suggested(), serializedCluster.suggested());
             assertEquals(originalCluster.target(), serializedCluster.target());
+            assertEquals(originalCluster.clusterInfo(), serializedCluster.clusterInfo());
             assertEquals(originalCluster.bcpGroupInfo(), serializedCluster.bcpGroupInfo());
             assertEquals(originalCluster.scalingEvents(), serializedCluster.scalingEvents());
         }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
index 5beb4b11c0c..a24eb61bb79 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/DynamicProvisioningTester.java
@@ -153,6 +153,7 @@ public class DynamicProvisioningTester {
                               cluster.required(),
                               cluster.suggested(),
                               cluster.target(),
+                              cluster.clusterInfo(),
                               cluster.bcpGroupInfo(),
                               List.of()); // Remove scaling events
         cluster = cluster.with(ScalingEvent.create(cluster.minResources(), cluster.minResources(),