Merge pull request #27796 from vespa-engine/mpolden/tweak-throttling

Tweak throttling

4 files changed, 26 insertions, 19 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
index 377ba11d170..8a9a29f58c6 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
@@ -68,7 +68,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
         super(nodeRepository, interval, metric);
         this.hostProvisioner = hostProvisioner;
         this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource);
-        this.throttler = new ProvisioningThrottler(nodeRepository.clock(), metric);
+        this.throttler = new ProvisioningThrottler(nodeRepository, metric);
     }
 
     @Override
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
index 6db8701041e..79b1bccbbde 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
@@ -43,7 +43,7 @@ public class Preparer {
         this.nodeRepository = nodeRepository;
         this.hostProvisioner = hostProvisioner;
         this.loadBalancerProvisioner = loadBalancerProvisioner;
-        this.throttler = new ProvisioningThrottler(nodeRepository.clock(), metric);
+        this.throttler = new ProvisioningThrottler(nodeRepository, metric);
     }
 
     /**
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java
index a1d4668acf3..b08e7dbccb0 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java
@@ -4,10 +4,10 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.History;
 
-import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Objects;
@@ -28,36 +28,40 @@ public class ProvisioningThrottler {
     private static final int MIN_SIZE = 100;
     private static final int MAX_GROWTH = 200;
     private static final double MAX_GROWTH_RATE = 0.4;
-    private static final Duration WINDOW = Duration.ofHours(8);
 
-    private final Clock clock;
+    private final NodeRepository nodeRepository;
     private final Metric metric;
 
-    public ProvisioningThrottler(Clock clock, Metric metric) {
-        this.clock = Objects.requireNonNull(clock);
+    public ProvisioningThrottler(NodeRepository nodeRepository, Metric metric) {
+        this.nodeRepository = Objects.requireNonNull(nodeRepository);
         this.metric = Objects.requireNonNull(metric);
     }
 
+    private Duration window() {
+        return nodeRepository.zone().system().isCd() ? Duration.ofHours(2) : Duration.ofHours(8);
+    }
+
     /** Returns whether provisioning should be throttled at given instant */
     public boolean throttle(NodeList allNodes, Agent agent) {
-        Instant startOfWindow = clock.instant().minus(WINDOW);
+        Duration window = window();
+        Instant startOfWindow = nodeRepository.clock().instant().minus(window);
         NodeList hosts = allNodes.hosts();
         int existingHosts = hosts.not().state(Node.State.deprovisioned).size();
         int provisionedRecently = hosts.matching(host -> host.history().hasEventAfter(History.Event.Type.provisioned, startOfWindow))
                                        .size();
-        boolean throttle = throttle(provisionedRecently, existingHosts, agent);
+        boolean throttle = throttle(provisionedRecently, existingHosts, window, agent);
         metric.set(throttlingActiveMetric, throttle ? 1 : 0, null);
         return throttle;
     }
 
-    static boolean throttle(int recent, int total, Agent agent) {
-        if (total < MIN_SIZE && recent < MIN_SIZE) return false; // Allow burst in small zones
+    static boolean throttle(int recent, int total, Duration window, Agent agent) {
+        if (total < MIN_SIZE && recent < MAX_GROWTH) return false; // Allow burst in small zones
         int maxGrowth = Math.min(MAX_GROWTH, (int) (total * MAX_GROWTH_RATE));
         boolean throttle = recent > maxGrowth;
         if (throttle) {
             LOG.warning(String.format("Throttling provisioning of new hosts by %s: %d hosts have been provisioned " +
                                       "in the past %s, which exceeds growth limit of %d", agent,
-                                      recent, WINDOW, maxGrowth));
+                                      recent, window, maxGrowth));
         }
         return throttle;
     }
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java
index 66c17e1d37e..f38b4732ed7 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java
@@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.provision.provisioning;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import org.junit.jupiter.api.Test;
 
+import java.time.Duration;
+
 import static org.junit.jupiter.api.Assertions.assertFalse;
 import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningThrottler.throttle;
 import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -15,13 +17,14 @@ class ProvisioningThrottlerTest {
     @Test
     void throttling() {
         Agent agent = Agent.system;
-        assertFalse(throttle(99, 99, agent));
-        assertTrue(throttle(100, 99, agent));
-        assertFalse(throttle(40, 100, agent));
-        assertTrue(throttle(41, 100, agent));
-        assertTrue(throttle(100, 100, agent));
-        assertFalse(throttle(200, 2100, agent));
-        assertTrue(throttle(201, 2100, agent));
+        Duration window = Duration.ofHours(1);
+        assertFalse(throttle(199, 99, window, agent));
+        assertTrue(throttle(200, 99, window, agent));
+        assertFalse(throttle(40, 100, window, agent));
+        assertTrue(throttle(41, 100, window, agent));
+        assertTrue(throttle(100, 100, window, agent));
+        assertFalse(throttle(200, 2100, window, agent));
+        assertTrue(throttle(201, 2100, window, agent));
     }
 
 }