From 697356065a407b10cfd69c58bad8fe2eed07e9e9 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 26 Jul 2023 10:49:25 +0200 Subject: Verify selected flavor is within resource limits --- .../hosted/provision/maintenance/HostCapacityMaintainer.java | 1 + .../hosted/provision/maintenance/HostFlavorUpgrader.java | 11 ++++++++--- .../vespa/hosted/provision/provisioning/HostProvisioner.java | 7 +++++-- .../yahoo/vespa/hosted/provision/provisioning/Preparer.java | 7 +++---- .../vespa/hosted/provision/testutils/MockHostProvisioner.java | 8 +++++--- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java index 8a9a29f58c6..2fa4fc82867 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostCapacityMaintainer.java @@ -224,6 +224,7 @@ public class HostCapacityMaintainer extends NodeRepositoryMaintainer { nodeRepository().zone().cloud().account(), false); List hosts = new ArrayList<>(); hostProvisioner.provisionHosts(request, + resources -> true, provisionedHosts -> { hosts.addAll(provisionedHosts.stream().map(host -> host.generateHost(Duration.ZERO)).toList()); nodeRepository().nodes().addNodes(hosts, Agent.HostCapacityMaintainer); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java index b16f2c5c17e..83a710cb5a9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java @@ -3,18 +3,21 @@ package com.yahoo.vespa.hosted.provision.maintenance; import com.yahoo.config.provision.Deployer; import com.yahoo.config.provision.NodeAllocationException; +import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Agent; +import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import java.time.Duration; import java.util.Objects; import java.util.Optional; import java.util.Random; +import java.util.function.Predicate; import java.util.logging.Level; /** @@ -63,16 +66,18 @@ public class HostFlavorUpgrader extends NodeRepositoryMaintainer { for (var node : activeNodes) { Optional parent = allNodes.parentOf(node); if (parent.isEmpty()) continue; - if (!hostProvisioner.canUpgradeFlavor(parent.get(), node)) continue; + Allocation allocation = node.allocation().get(); + Predicate realHostResourcesWithinLimits = resources -> nodeRepository().nodeResourceLimits().isWithinRealLimits(resources, allocation.owner(), allocation.membership().cluster()); + if (!hostProvisioner.canUpgradeFlavor(parent.get(), node, realHostResourcesWithinLimits)) continue; if (parent.get().status().wantToUpgradeFlavor()) continue; // Already upgrading boolean redeployed = false; boolean deploymentValid = false; - try (MaintenanceDeployment deployment = new MaintenanceDeployment(node.allocation().get().owner(), deployer, metric, nodeRepository(), true)) { + try (MaintenanceDeployment deployment = new MaintenanceDeployment(allocation.owner(), deployer, metric, nodeRepository(), true)) { deploymentValid = deployment.isValid(); if (!deploymentValid) continue; - log.log(Level.INFO, () -> "Redeploying " + node.allocation().get().owner() + " to upgrade flavor (" + + log.log(Level.INFO, () -> "Redeploying " + allocation.owner() + " to upgrade flavor (" + parent.get().flavor().name() + ") of " + parent.get()); upgradeFlavor(parent.get(), true); deployment.activate(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java index dd838375a59..66d1a4e8bc8 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java @@ -4,10 +4,12 @@ package com.yahoo.vespa.hosted.provision.provisioning; import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.HostEvent; import com.yahoo.config.provision.NodeAllocationException; +import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.hosted.provision.Node; import java.util.List; import java.util.function.Consumer; +import java.util.function.Predicate; /** * A service which supports provisioning container hosts dynamically. @@ -33,13 +35,14 @@ public interface HostProvisioner { * Schedule provisioning of a given number of hosts. * * @param request details of the host provision request. + * @param realHostResourcesWithinLimits predicate that returns true if the given resources are within allowed limits * @param whenProvisioned consumer of {@link ProvisionedHost}s describing the provisioned nodes, * the {@link Node} returned from {@link ProvisionedHost#generateHost} must be * written to ZK immediately in case the config server goes down while waiting * for the provisioning to finish. * @throws NodeAllocationException if the cloud provider cannot satisfy the request */ - void provisionHosts(HostProvisionRequest request, Consumer> whenProvisioned) throws NodeAllocationException; + void provisionHosts(HostProvisionRequest request, Predicate realHostResourcesWithinLimits, Consumer> whenProvisioned) throws NodeAllocationException; /** * Continue provisioning of given list of Nodes. @@ -75,6 +78,6 @@ public interface HostProvisioner { List hostEventsIn(List cloudAccounts); /** Returns whether flavor for given host can be upgraded to a newer generation */ - boolean canUpgradeFlavor(Node host, Node child); + boolean canUpgradeFlavor(Node host, Node child, Predicate realHostResourcesWithinLimits); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java index 470a267e75f..349be9e4b47 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/Preparer.java @@ -20,6 +20,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Optional; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; @@ -117,10 +118,8 @@ public class Preparer { Optional.of(cluster.id()), requested.cloudAccount(), deficit.dueToFlavorUpgrade()); - if (throttler.throttle(allNodes, Agent.system)) { - throw new NodeAllocationException("Host provisioning is being throttled", true); - } - hostProvisioner.get().provisionHosts(request, whenProvisioned); + Predicate realHostResourcesWithinLimits = resources -> nodeRepository.nodeResourceLimits().isWithinRealLimits(resources, application, cluster); + hostProvisioner.get().provisionHosts(request, realHostResourcesWithinLimits, whenProvisioned); } catch (NodeAllocationException e) { // Mark the nodes that were written to ZK in the consumer for deprovisioning. While these hosts do // not exist, we cannot remove them from ZK here because other nodes may already have been diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java index e6a064c7bf5..965611b9a6e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java @@ -29,6 +29,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.stream.IntStream; /** @@ -68,13 +69,14 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public void provisionHosts(HostProvisionRequest request, Consumer> whenProvisioned) { + public void provisionHosts(HostProvisionRequest request, Predicate realHostResourcesWithinLimits, Consumer> whenProvisioned) throws NodeAllocationException { if (behaviour(Behaviour.failProvisionRequest)) throw new NodeAllocationException("No capacity for provision request", true); Flavor hostFlavor = hostFlavors.get(request.clusterType().orElse(ClusterSpec.Type.content)); if (hostFlavor == null) hostFlavor = flavors.stream() .filter(f -> request.sharing() == HostSharing.exclusive ? compatible(f, request.resources()) : f.resources().satisfies(request.resources())) + .filter(f -> realHostResourcesWithinLimits.test(f.resources())) .findFirst() .orElseThrow(() -> new NodeAllocationException("No host flavor matches " + request.resources(), true)); @@ -130,7 +132,7 @@ public class MockHostProvisioner implements HostProvisioner { } @Override - public boolean canUpgradeFlavor(Node host, Node child) { + public boolean canUpgradeFlavor(Node host, Node child, Predicate realHostResourcesWithinLimits) { return upgradableFlavors.contains(host.flavor().name()); } @@ -248,7 +250,7 @@ public class MockHostProvisioner implements HostProvisioner { /** Fail call to {@link MockHostProvisioner#provision(com.yahoo.vespa.hosted.provision.Node)} */ failProvisioning, - /** Fail call to {@link MockHostProvisioner#provisionHosts(HostProvisionRequest, Consumer)} */ + /** Fail call to {@link MockHostProvisioner#provisionHosts(HostProvisionRequest, Predicate, Consumer)} */ failProvisionRequest, /** Fail call to {@link MockHostProvisioner#deprovision(com.yahoo.vespa.hosted.provision.Node)} */ -- cgit v1.2.3