diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-07-14 11:40:39 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-07-14 11:44:22 +0200 |
commit | 4388580fd96a6f0426559c35f3e126280ee2c204 (patch) | |
tree | b95facd328b861a384f17ee36ffcd4d702d2ffe6 /node-repository | |
parent | 3c1db11cae9f0723b164b19cdb0ac17c6170c671 (diff) |
Throttle host provisioning
Diffstat (limited to 'node-repository')
8 files changed, 120 insertions, 11 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 8213286639c..377ba11d170 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner.HostSharing import com.yahoo.vespa.hosted.provision.provisioning.NodeCandidate; import com.yahoo.vespa.hosted.provision.provisioning.NodePrioritizer; import com.yahoo.vespa.hosted.provision.provisioning.NodeSpec; +import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningThrottler; import java.time.Duration; import java.time.Instant; @@ -57,6 +58,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { private final HostProvisioner hostProvisioner; private final ListFlag<ClusterCapacity> preprovisionCapacityFlag; + private final ProvisioningThrottler throttler; HostCapacityMaintainer(NodeRepository nodeRepository, Duration interval, @@ -66,6 +68,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { super(nodeRepository, interval, metric); this.hostProvisioner = hostProvisioner; this.preprovisionCapacityFlag = PermanentFlags.PREPROVISION_CAPACITY.bindTo(flagSource); + this.throttler = new ProvisioningThrottler(nodeRepository.clock(), metric); } @Override @@ -203,19 +206,23 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { var clusterType = Optional.ofNullable(clusterCapacityDeficit.clusterType()); nodesPlusProvisioned.addAll(provisionHosts(clusterCapacityDeficit.count(), toNodeResources(clusterCapacityDeficit), - clusterType.map(ClusterSpec.Type::from))); + clusterType.map(ClusterSpec.Type::from), + nodeList)); } } - private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<ClusterSpec.Type> clusterType) { + private List<Node> provisionHosts(int count, NodeResources nodeResources, Optional<ClusterSpec.Type> clusterType, NodeList allNodes) { try { + if (throttler.throttle(allNodes, Agent.HostCapacityMaintainer)) { + throw new NodeAllocationException("Host provisioning is being throttled", true); + } Version osVersion = nodeRepository().osVersions().targetFor(NodeType.host).orElse(Version.emptyVersion); List<Integer> provisionIndices = nodeRepository().database().readProvisionIndices(count); - List<Node> hosts = new ArrayList<>(); HostProvisionRequest request = new HostProvisionRequest(provisionIndices, NodeType.host, nodeResources, ApplicationId.defaultId(), osVersion, HostSharing.shared, clusterType, Optional.empty(), nodeRepository().zone().cloud().account(), false); + List<Node> hosts = new ArrayList<>(); hostProvisioner.provisionHosts(request, provisionedHosts -> { hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java index 43b8cd08989..bcc63a6704a 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/NodeRepositoryProvisioner.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.ProvisionLock; import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -61,7 +62,8 @@ public class NodeRepositoryProvisioner implements Provisioner { @Inject public NodeRepositoryProvisioner(NodeRepository nodeRepository, Zone zone, - ProvisionServiceProvider provisionServiceProvider) { + ProvisionServiceProvider provisionServiceProvider, + Metric metric) { this.nodeRepository = nodeRepository; this.allocationOptimizer = new AllocationOptimizer(nodeRepository); this.capacityPolicies = new CapacityPolicies(nodeRepository); @@ -71,7 +73,8 @@ public class NodeRepositoryProvisioner implements Provisioner { this.nodeResourceLimits = new NodeResourceLimits(nodeRepository); this.preparer = new Preparer(nodeRepository, provisionServiceProvider.getHostProvisioner(), - loadBalancerProvisioner); + loadBalancerProvisioner, + metric); this.activator = new Activator(nodeRepository, loadBalancerProvisioner); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 8975dda8e60..6db8701041e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.NodeAllocationException; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; +import com.yahoo.jdisc.Metric; import com.yahoo.transaction.Mutex; import com.yahoo.vespa.hosted.provision.LockedNodeList; import com.yahoo.vespa.hosted.provision.Node; @@ -36,11 +37,13 @@ public class Preparer { private final NodeRepository nodeRepository; private final Optional<HostProvisioner> hostProvisioner; private final Optional<LoadBalancerProvisioner> loadBalancerProvisioner; + private final ProvisioningThrottler throttler; - public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner) { + public Preparer(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, Optional<LoadBalancerProvisioner> loadBalancerProvisioner, Metric metric) { this.nodeRepository = nodeRepository; this.hostProvisioner = hostProvisioner; this.loadBalancerProvisioner = loadBalancerProvisioner; + this.throttler = new ProvisioningThrottler(nodeRepository.clock(), metric); } /** @@ -110,6 +113,9 @@ public class Preparer { Optional.of(cluster.id()), requested.cloudAccount(), deficit.dueToFlavorUpgrade()); + if (throttler.throttle(allNodes, Agent.system)) { + throw new NodeAllocationException("Host provisioning is being throttled", true); + } hostProvisioner.get().provisionHosts(request, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java new file mode 100644 index 00000000000..a1d4668acf3 --- /dev/null +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottler.java @@ -0,0 +1,65 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.jdisc.Metric; +import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.History; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.util.Objects; +import java.util.logging.Logger; + +/** + * Throttles provisioning of new hosts in dynamically provisioned zones. + * + * @author mpolden + */ +public class ProvisioningThrottler { + + /** Metric that indicates whether throttling is active where 1 means active and 0 means inactive */ + private static final String throttlingActiveMetric = "throttledHostProvisioning"; + + private static final Logger LOG = Logger.getLogger(ProvisioningThrottler.class.getName()); + + private static final int MIN_SIZE = 100; + private static final int MAX_GROWTH = 200; + private static final double MAX_GROWTH_RATE = 0.4; + private static final Duration WINDOW = Duration.ofHours(8); + + private final Clock clock; + private final Metric metric; + + public ProvisioningThrottler(Clock clock, Metric metric) { + this.clock = Objects.requireNonNull(clock); + this.metric = Objects.requireNonNull(metric); + } + + /** Returns whether provisioning should be throttled at given instant */ + public boolean throttle(NodeList allNodes, Agent agent) { + Instant startOfWindow = clock.instant().minus(WINDOW); + NodeList hosts = allNodes.hosts(); + int existingHosts = hosts.not().state(Node.State.deprovisioned).size(); + int provisionedRecently = hosts.matching(host -> host.history().hasEventAfter(History.Event.Type.provisioned, startOfWindow)) + .size(); + boolean throttle = throttle(provisionedRecently, existingHosts, agent); + metric.set(throttlingActiveMetric, throttle ? 1 : 0, null); + return throttle; + } + + static boolean throttle(int recent, int total, Agent agent) { + if (total < MIN_SIZE && recent < MIN_SIZE) return false; // Allow burst in small zones + int maxGrowth = Math.min(MAX_GROWTH, (int) (total * MAX_GROWTH_RATE)); + boolean throttle = recent > maxGrowth; + if (throttle) { + LOG.warning(String.format("Throttling provisioning of new hosts by %s: %d hosts have been provisioned " + + "in the past %s, which exceeds growth limit of %d", agent, + recent, WINDOW, maxGrowth)); + } + return throttle; + } + +} diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java index 40460e70861..26478d2b566 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockNodeRepository.java @@ -25,6 +25,7 @@ import com.yahoo.config.provision.Zone; import com.yahoo.config.provision.ZoneEndpoint; import com.yahoo.config.provision.ZoneEndpoint.AccessType; import com.yahoo.config.provision.ZoneEndpoint.AllowedUrn; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.curator.mock.MockCurator; @@ -104,7 +105,7 @@ public class MockNodeRepository extends NodeRepository { } private void populate() { - NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider()); + NodeRepositoryProvisioner provisioner = new NodeRepositoryProvisioner(this, Zone.defaultZone(), new MockProvisionServiceProvider(), new MockMetric()); List<Node> nodes = new ArrayList<>(); // Regular nodes diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java index 7763459dd92..79644206918 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/InfraDeployerImplTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.Provisioner; import com.yahoo.config.provision.Zone; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -60,7 +61,7 @@ public class InfraDeployerImplTest { private final NodeRepositoryTester tester = new NodeRepositoryTester(); private final NodeRepository nodeRepository = tester.nodeRepository(); - private final Provisioner provisioner = spy(new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new EmptyProvisionServiceProvider())); + private final Provisioner provisioner = spy(new NodeRepositoryProvisioner(nodeRepository, Zone.defaultZone(), new EmptyProvisionServiceProvider(), new MockMetric())); private final InfrastructureVersions infrastructureVersions = nodeRepository.infrastructureVersions(); private final DuperModelInfraApi duperModelInfraApi = mock(DuperModelInfraApi.class); private final InfraDeployerImpl infraDeployer; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java index a3a90d58c2c..bca48b19ccf 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningTester.java @@ -22,12 +22,12 @@ import com.yahoo.config.provision.NodeResources.DiskSpeed; import com.yahoo.config.provision.NodeResources.StorageType; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.ProvisionLock; -import com.yahoo.config.provision.ProvisionLogger; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.Zone; import com.yahoo.config.provisioning.FlavorsConfig; +import com.yahoo.jdisc.test.MockMetric; import com.yahoo.test.ManualClock; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.applicationmodel.InfrastructureApplication; @@ -73,7 +73,6 @@ import java.util.UUID; import java.util.function.Function; import java.util.function.Predicate; import java.util.function.UnaryOperator; -import java.util.logging.Level; import java.util.stream.Collectors; import static com.yahoo.config.provision.NodeResources.StorageType.local; @@ -131,7 +130,7 @@ public class ProvisioningTester { true, spareCount, 1000); - this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider); + this.provisioner = new NodeRepositoryProvisioner(nodeRepository, zone, provisionServiceProvider, new MockMetric()); this.capacityPolicies = new CapacityPolicies(nodeRepository); this.provisionLogger = new InMemoryProvisionLogger(); this.loadBalancerService = loadBalancerService; diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java new file mode 100644 index 00000000000..66c17e1d37e --- /dev/null +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/provisioning/ProvisioningThrottlerTest.java @@ -0,0 +1,27 @@ +package com.yahoo.vespa.hosted.provision.provisioning; + +import com.yahoo.vespa.hosted.provision.node.Agent; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static com.yahoo.vespa.hosted.provision.provisioning.ProvisioningThrottler.throttle; +import static org.junit.jupiter.api.Assertions.assertTrue; + +/** + * @author mpolden + */ +class ProvisioningThrottlerTest { + + @Test + void throttling() { + Agent agent = Agent.system; + assertFalse(throttle(99, 99, agent)); + assertTrue(throttle(100, 99, agent)); + assertFalse(throttle(40, 100, agent)); + assertTrue(throttle(41, 100, agent)); + assertTrue(throttle(100, 100, agent)); + assertFalse(throttle(200, 2100, agent)); + assertTrue(throttle(201, 2100, agent)); + } + +} |