Throttle host provisioning

8 files changed, 120 insertions, 11 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
index 8213286639c..377ba11d170 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java
@@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing
 import com.yahoo.vespa.hosted.provision.provisioning.NodeCandidate;
 import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer;
 import com.yahoo.vespa.hosted.provision.provisioning.NodeSpec;
+import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningThrottler;
 
 import java.time.Duration;
 import java.time.Instant;
@@ -57,6 +58,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
 
     private final HostProvisioner hostProvisioner;
     private final ListFlag<ClusterCapacity> preprovisionCapacityFlag;
+    private final ProvisioningThrottler throttler;
 
     HostCapacityMaintainer(NodeRepository nodeRepository,
                            Duration interval,
@@ -66,6 +68,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
         super(nodeRepository, interval, metric);
         this.hostProvisioner = hostProvisioner;
         this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource);
+        this.throttler = new ProvisioningThrottler(nodeRepository.clock(), metric);
     }
 
     @Override
@@ -203,19 +206,23 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer {
             var clusterType = Optional.ofNullable(clusterCapacityDeficit.clusterType());
             nodesPlusProvisioned.addAll(provisionHosts(clusterCapacityDeficit.count(),
                                                        toNodeResources(clusterCapacityDeficit),
-                                                       clusterType.map(ClusterSpec.Type::from)));
+                                                       clusterType.map(ClusterSpec.Type::from),
+                                                       nodeList));
         }
     }
 
-    private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<ClusterSpec.Type> clusterType) {
+    private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<ClusterSpec.Type> clusterType, NodeList allNodes) {
         try {
+            if (throttler.throttle(allNodes, Agent.HostCapacityMaintainer)) {
+                throw new NodeAllocationException("Host provisioning is being throttled", true);
+            }
             Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion);
             List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count);
-            List<Node> hosts = new ArrayList<>();
             HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources,
                                                                     ApplicationId.defaultId(), osVersion,
                                                                     HostSharing.shared, clusterType, Optional.empty(),
                                                                     nodeRepository().zone().cloud().account(), false);
+            List<Node> hosts = new ArrayList<>();
             hostProvisioner.provisionHosts(request,
                                            provisionedHosts -> {
                                                hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList());
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
index 43b8cd08989..bcc63a6704a 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java
@@ -16,6 +16,7 @@ import com.yahoo.config.provision.ProvisionLock;
 import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
@@ -61,7 +62,8 @@ public class NodeRepositoryProvisioner implements Provisioner {
     @Inject
     public NodeRepositoryProvisioner(NodeRepository nodeRepository,
                                      Zone zone,
-                                     ProvisionServiceProvider provisionServiceProvider) {
+                                     ProvisionServiceProvider provisionServiceProvider,
+                                     Metric metric) {
         this.nodeRepository = nodeRepository;
         this.allocationOptimizer = new AllocationOptimizer(nodeRepository);
         this.capacityPolicies = new CapacityPolicies(nodeRepository);
@@ -71,7 +73,8 @@ public class NodeRepositoryProvisioner implements Provisioner {
         this.nodeResourceLimits = new NodeResourceLimits(nodeRepository);
         this.preparer = new Preparer(nodeRepository,
                                      provisionServiceProvider.getHostProvisioner(),
-                                     loadBalancerProvisioner);
+                                     loadBalancerProvisioner,
+                                     metric);
         this.activator = new Activator(nodeRepository, loadBalancerProvisioner);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
index 8975dda8e60..6db8701041e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java
@@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.NodeAllocationException;
 import com.yahoo.config.provision.NodeResources;
 import com.yahoo.config.provision.NodeType;
+import com.yahoo.jdisc.Metric;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.vespa.hosted.provision.LockedNodeList;
 import com.yahoo.vespa.hosted.provision.Node;
@@ -36,11 +37,13 @@ public class Preparer {
     private final NodeRepository nodeRepository;
     private final Optional<HostProvisioner> hostProvisioner;
     private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner;
+    private final ProvisioningThrottler throttler;
 
-    public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner) {
+    public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner, Metric metric) {
         this.nodeRepository = nodeRepository;
         this.hostProvisioner = hostProvisioner;
         this.loadBalancerProvisioner = loadBalancerProvisioner;
+        this.throttler = new ProvisioningThrottler(nodeRepository.clock(), metric);
     }
 
     /**
@@ -110,6 +113,9 @@ public class Preparer {
                                                                             Optional.of(cluster.id()),
                                                                             requested.cloudAccount(),
                                                                             deficit.dueToFlavorUpgrade());
+                    if (throttler.throttle(allNodes, Agent.system)) {
+                        throw new NodeAllocationException("Host provisioning is being throttled", true);
+                    }
                     hostProvisioner.get().provisionHosts(request, whenProvisioned);
                 } catch (NodeAllocationException e) {
                     // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java
new file mode 100644
index 00000000000..a1d4668acf3
--- /dev/null
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java
@@ -0,0 +1,65 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.jdisc.Metric;
+import com.yahoo.vespa.hosted.provision.Node;
+import com.yahoo.vespa.hosted.provision.NodeList;
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import com.yahoo.vespa.hosted.provision.node.History;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Objects;
+import java.util.logging.Logger;
+
+/**
+ * Throttles provisioning of new hosts in dynamically provisioned zones.
+ *
+ * @author mpolden
+ */
+public class ProvisioningThrottler {
+
+    /** Metric that indicates whether throttling is active where 1 means active and 0 means inactive */
+    private static final String throttlingActiveMetric = "throttledHostProvisioning";
+
+    private static final Logger LOG = Logger.getLogger(ProvisioningThrottler.class.getName());
+
+    private static final int MIN_SIZE = 100;
+    private static final int MAX_GROWTH = 200;
+    private static final double MAX_GROWTH_RATE = 0.4;
+    private static final Duration WINDOW = Duration.ofHours(8);
+
+    private final Clock clock;
+    private final Metric metric;
+
+    public ProvisioningThrottler(Clock clock, Metric metric) {
+        this.clock = Objects.requireNonNull(clock);
+        this.metric = Objects.requireNonNull(metric);
+    }
+
+    /** Returns whether provisioning should be throttled at given instant */
+    public boolean throttle(NodeList allNodes, Agent agent) {
+        Instant startOfWindow = clock.instant().minus(WINDOW);
+        NodeList hosts = allNodes.hosts();
+        int existingHosts = hosts.not().state(Node.State.deprovisioned).size();
+        int provisionedRecently = hosts.matching(host -> host.history().hasEventAfter(History.Event.Type.provisioned, startOfWindow))
+                                       .size();
+        boolean throttle = throttle(provisionedRecently, existingHosts, agent);
+        metric.set(throttlingActiveMetric, throttle ? 1 : 0, null);
+        return throttle;
+    }
+
+    static boolean throttle(int recent, int total, Agent agent) {
+        if (total < MIN_SIZE && recent < MIN_SIZE) return false; // Allow burst in small zones
+        int maxGrowth = Math.min(MAX_GROWTH, (int) (total * MAX_GROWTH_RATE));
+        boolean throttle = recent > maxGrowth;
+        if (throttle) {
+            LOG.warning(String.format("Throttling provisioning of new hosts by %s: %d hosts have been provisioned " +
+                                      "in the past %s, which exceeds growth limit of %d", agent,
+                                      recent, WINDOW, maxGrowth));
+        }
+        return throttle;
+    }
+
+}
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
index 40460e70861..26478d2b566 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java
@@ -25,6 +25,7 @@ import com.yahoo.config.provision.Zone;
 import com.yahoo.config.provision.ZoneEndpoint;
 import com.yahoo.config.provision.ZoneEndpoint.AccessType;
 import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn;
+import com.yahoo.jdisc.test.MockMetric;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.curator.mock.MockCurator;
@@ -104,7 +105,7 @@ public class MockNodeRepository extends NodeRepository {
     }
 
     private void populate() {
-        NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider());
+        NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider(), new MockMetric());
         List<Node> nodes = new ArrayList<>();
 
         // Regular nodes
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
index 7763459dd92..79644206918 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.HostSpec;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.Provisioner;
 import com.yahoo.config.provision.Zone;
+import com.yahoo.jdisc.test.MockMetric;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
@@ -60,7 +61,7 @@ public class InfraDeployerImplTest {
 
     private final NodeRepositoryTester tester = new NodeRepositoryTester();
     private final NodeRepository nodeRepository = tester.nodeRepository();
-    private final Provisioner provisioner = spy(new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new EmptyProvisionServiceProvider()));
+    private final Provisioner provisioner = spy(new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new EmptyProvisionServiceProvider(), new MockMetric()));
     private final InfrastructureVersions infrastructureVersions = nodeRepository.infrastructureVersions();
     private final DuperModelInfraApi duperModelInfraApi = mock(DuperModelInfraApi.class);
     private final InfraDeployerImpl infraDeployer;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
index a3a90d58c2c..bca48b19ccf 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java
@@ -22,12 +22,12 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed;
 import com.yahoo.config.provision.NodeResources.StorageType;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.ProvisionLock;
-import com.yahoo.config.provision.ProvisionLogger;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.config.provisioning.FlavorsConfig;
+import com.yahoo.jdisc.test.MockMetric;
 import com.yahoo.test.ManualClock;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.applicationmodel.InfrastructureApplication;
@@ -73,7 +73,6 @@ import java.util.UUID;
 import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.function.UnaryOperator;
-import java.util.logging.Level;
 import java.util.stream.Collectors;
 
 import static com.yahoo.config.provision.NodeResources.StorageType.local;
@@ -131,7 +130,7 @@ public class ProvisioningTester {
                                                  true,
                                                  spareCount,
                                                  1000);
-        this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider);
+        this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, new MockMetric());
         this.capacityPolicies = new CapacityPolicies(nodeRepository);
         this.provisionLogger = new InMemoryProvisionLogger();
         this.loadBalancerService = loadBalancerService;
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java
new file mode 100644
index 00000000000..66c17e1d37e
--- /dev/null
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java
@@ -0,0 +1,27 @@
+package com.yahoo.vespa.hosted.provision.provisioning;
+
+import com.yahoo.vespa.hosted.provision.node.Agent;
+import org.junit.jupiter.api.Test;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningThrottler.throttle;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * @author mpolden
+ */
+class ProvisioningThrottlerTest {
+
+    @Test
+    void throttling() {
+        Agent agent = Agent.system;
+        assertFalse(throttle(99, 99, agent));
+        assertTrue(throttle(100, 99, agent));
+        assertFalse(throttle(40, 100, agent));
+        assertTrue(throttle(41, 100, agent));
+        assertTrue(throttle(100, 100, agent));
+        assertFalse(throttle(200, 2100, agent));
+        assertTrue(throttle(201, 2100, agent));
+    }
+
+}