diff options
author | Valerij Fredriksen <valerijf@vespa.ai> | 2023-11-01 17:19:42 +0100 |
---|---|---|
committer | Valerij Fredriksen <valerijf@vespa.ai> | 2023-11-02 11:31:39 +0100 |
commit | 2d8d4b19a6bc48dd93403deb2a1a1732943d4249 (patch) | |
tree | 15c98c644901ed22dd9b412c225860c1d2af42e3 /node-repository | |
parent | ebb157dc5353e6f7e16001165bf13b738ea35142 (diff) |
HostFlavorUpgrader: Reprocess hosts with non-retiring nodes
Diffstat (limited to 'node-repository')
2 files changed, 13 insertions, 4 deletions
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java index 01da75f90b6..b6897d5b1c9 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgrader.java @@ -14,9 +14,11 @@ import com.yahoo.vespa.hosted.provision.node.Allocation; import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner; import java.time.Duration; +import java.util.HashSet; import java.util.Objects; import java.util.Optional; import java.util.Random; +import java.util.Set; import java.util.function.Predicate; import java.util.logging.Level; @@ -63,13 +65,15 @@ public class HostFlavorUpgrader extends NodeRepositoryMaintainer { NodeList activeNodes = allNodes.nodeType(NodeType.tenant) .state(Node.State.active) .shuffle(random); // Shuffle to avoid getting stuck trying to upgrade the same host + Set<String> exhaustedFlavors = new HashSet<>(); for (var node : activeNodes) { Optional<Node> parent = allNodes.parentOf(node); if (parent.isEmpty()) continue; + if (exhaustedFlavors.contains(parent.get().flavor().name())) continue; Allocation allocation = node.allocation().get(); Predicate<NodeResources> realHostResourcesWithinLimits = resources -> nodeRepository().nodeResourceLimits().isWithinRealLimits(resources, allocation.owner(), allocation.membership().cluster()); if (!hostProvisioner.canUpgradeFlavor(parent.get(), node, realHostResourcesWithinLimits)) continue; - if (parent.get().status().wantToUpgradeFlavor()) continue; // Already upgrading + if (parent.get().status().wantToUpgradeFlavor() && allocation.membership().retired()) continue; // Already upgrading boolean redeployed = false; boolean deploymentValid = false; @@ -85,6 +89,7 @@ public class HostFlavorUpgrader extends NodeRepositoryMaintainer { return 1.0; } catch (NodeAllocationException e) { // Fine, no capacity for upgrade + exhaustedFlavors.add(parent.get().flavor().name()); } finally { if (deploymentValid && !redeployed) { // Cancel upgrade if redeploy failed upgradeFlavor(parent.get(), false); diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgraderTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgraderTest.java index e4c12ede1d9..64c0135bd8e 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgraderTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/maintenance/HostFlavorUpgraderTest.java @@ -68,12 +68,16 @@ class HostFlavorUpgraderTest { .matching(node -> node.status().wantToUpgradeFlavor() || node.status().wantToRetire()), "No hosts marked for upgrade or retirement"); - // First provision request fails, but second succeeds and a replacement host starts provisioning + // First provision request fails, but we only try once for the same flavor hostProvisioner.with(Behaviour.failProvisionRequest, 1); assertEquals(1, upgrader.maintain()); NodeList nodes = tester.nodeRepository().nodes().list(); - NodeList upgradingFlavor = nodes.matching(node -> node.status().wantToRetire() && - node.status().wantToUpgradeFlavor()); + assertEquals(0, nodes.matching(node -> node.status().wantToUpgradeFlavor()).size()); + + // Second succeeds and a replacement host starts provisioning + assertEquals(1, upgrader.maintain()); + nodes = tester.nodeRepository().nodes().list(); + NodeList upgradingFlavor = nodes.matching(node -> node.status().wantToUpgradeFlavor()); assertEquals(1, upgradingFlavor.size()); assertEquals(1, nodes.state(Node.State.provisioned).size()); |