diff options
9 files changed, 87 insertions, 100 deletions
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 0ccdfddf1b8..2f8efd5a717 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -358,13 +358,6 @@ public class Flags { "Set mode for CSRF filter ('disabled', 'log_only', 'enabled')", "Takes effect on controller restart/redeployment"); - public static final UnboundBooleanFlag SOFT_REBUILD = defineFeatureFlag( - "soft-rebuild", true, - List.of("mpolden"), "2022-09-27", "2022-12-01", - "Whether soft rebuild can be used to rebuild hosts with remote disk", - "Takes effect on next run of OsUpgradeActivator" - ); - public static final UnboundListFlag<String> CSRF_USERS = defineListFlag( "csrf-users", List.of(), String.class, List.of("bjorncs", "tokle"), "2022-09-22", "2023-06-01", diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java index e9aa72b986d..7470285849f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java @@ -183,10 +183,10 @@ public class PermanentFlags { "Takes effect on redeployment", APPLICATION_ID); - public static final UnboundIntFlag MAX_REBUILDS = defineIntFlag( - "max-host-rebuilds", 10, - "The maximum number of hosts allowed to rebuild at a time", - "Takes effect immediately, but any current excess rebuilds will not be cancelled" + public static final UnboundIntFlag MAX_OS_UPGRADES = defineIntFlag( + "max-os-upgrades", 30, + "The maximum number hosts that can perform OS upgrade at a time", + "Takes effect immediately, but any current excess upgrades will not be cancelled" ); public static final UnboundListFlag<String> EXTENDED_TRIAL_TENANTS = defineListFlag( diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java index 7aaf37a8ee6..6bab393945b 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java @@ -1,6 +1,7 @@ package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.hosted.provision.NodeRepository; import java.util.List; @@ -9,9 +10,12 @@ import java.util.List; * * @author mpolden */ -public record CompositeOsUpgrader(List<OsUpgrader> upgraders) implements OsUpgrader { +public class CompositeOsUpgrader extends OsUpgrader { - public CompositeOsUpgrader(List<OsUpgrader> upgraders) { + private final List<OsUpgrader> upgraders; + + public CompositeOsUpgrader(NodeRepository nodeRepository, List<OsUpgrader> upgraders) { + super(nodeRepository); this.upgraders = List.copyOf(upgraders); } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java index 4178d4a6328..23aa03a5315 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java @@ -8,7 +8,6 @@ import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter; import java.time.Instant; -import java.util.Objects; import java.util.Optional; import java.util.logging.Logger; @@ -20,32 +19,23 @@ import java.util.logging.Logger; * * @author mpolden */ -public class DelegatingOsUpgrader implements OsUpgrader { +public class DelegatingOsUpgrader extends OsUpgrader { private static final Logger LOG = Logger.getLogger(DelegatingOsUpgrader.class.getName()); - private final NodeRepository nodeRepository; - - /** The maximum number of nodes, within a single node type, that can upgrade in parallel. */ - private final int maxActiveUpgrades; - - public DelegatingOsUpgrader(NodeRepository nodeRepository, int maxActiveUpgrades) { - this.nodeRepository = Objects.requireNonNull(nodeRepository); - this.maxActiveUpgrades = maxActiveUpgrades; - if (maxActiveUpgrades < 1) throw new IllegalArgumentException("maxActiveUpgrades must be positive, was " + - maxActiveUpgrades); + public DelegatingOsUpgrader(NodeRepository nodeRepository) { + super(nodeRepository); } @Override public void upgradeTo(OsVersionTarget target) { NodeList activeNodes = nodeRepository.nodes().list(Node.State.active).nodeType(target.nodeType()); - int numberToUpgrade = Math.max(0, maxActiveUpgrades - activeNodes.changingOsVersionTo(target.version()).size()); Instant now = nodeRepository.clock().instant(); NodeList nodesToUpgrade = activeNodes.not().changingOsVersionTo(target.version()) .osVersionIsBefore(target.version()) .matching(node -> canUpgradeAt(now, node)) .byIncreasingOsVersion() - .first(numberToUpgrade); + .first(upgradeSlots(target, activeNodes)); if (nodesToUpgrade.size() == 0) return; LOG.info("Upgrading " + nodesToUpgrade.size() + " nodes of type " + target.nodeType() + " to OS version " + target.version().toFullString()); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java index 4140de76368..258609f043e 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java @@ -2,7 +2,11 @@ package com.yahoo.vespa.hosted.provision.os; import com.yahoo.config.provision.NodeType; +import com.yahoo.vespa.flags.IntFlag; +import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; +import com.yahoo.vespa.hosted.provision.NodeList; +import com.yahoo.vespa.hosted.provision.NodeRepository; import java.time.Instant; @@ -11,16 +15,35 @@ import java.time.Instant; * * @author mpolden */ -public interface OsUpgrader { +public abstract class OsUpgrader { + + private final IntFlag maxActiveUpgrades; + + final NodeRepository nodeRepository; + + public OsUpgrader(NodeRepository nodeRepository) { + this.nodeRepository = nodeRepository; + this.maxActiveUpgrades = PermanentFlags.MAX_OS_UPGRADES.bindTo(nodeRepository.flagSource()); + } /** Trigger upgrade to given target */ - void upgradeTo(OsVersionTarget target); + abstract void upgradeTo(OsVersionTarget target); /** Disable OS upgrade for all nodes of given type */ - void disableUpgrade(NodeType type); + abstract void disableUpgrade(NodeType type); + + /** Returns the number of upgrade slots available for given target */ + final int upgradeSlots(OsVersionTarget target, NodeList activeNodes) { + if (!activeNodes.stream().allMatch(node -> node.type() == target.nodeType())) { + throw new IllegalArgumentException("All node types must type of OS version target " + target.nodeType()); + } + int max = target.nodeType() == NodeType.host ? maxActiveUpgrades.value() : 1; + int upgrading = activeNodes.changingOsVersionTo(target.version()).size(); + return Math.max(0, max - upgrading); + } /** Returns whether node can upgrade at given instant */ - default boolean canUpgradeAt(Instant instant, Node node) { + boolean canUpgradeAt(Instant instant, Node node) { return true; } diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java index dda68735fbb..2939b940486 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java @@ -6,8 +6,6 @@ import com.yahoo.config.provision.Cloud; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.NodeType; import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeRepository; import com.yahoo.vespa.hosted.provision.node.Status; @@ -34,24 +32,17 @@ public class OsVersions { private static final Logger log = Logger.getLogger(OsVersions.class.getName()); - /** The maximum number of concurrent upgrades per node type */ - private static final int MAX_ACTIVE_UPGRADES = 30; - private final NodeRepository nodeRepository; private final CuratorDatabaseClient db; - private final int maxActiveUpgrades; - private final BooleanFlag softRebuildFlag; private final Cloud cloud; public OsVersions(NodeRepository nodeRepository) { - this(nodeRepository, nodeRepository.zone().cloud(), MAX_ACTIVE_UPGRADES); + this(nodeRepository, nodeRepository.zone().cloud()); } - OsVersions(NodeRepository nodeRepository, Cloud cloud, int maxActiveUpgrades) { + OsVersions(NodeRepository nodeRepository, Cloud cloud) { this.nodeRepository = Objects.requireNonNull(nodeRepository); this.db = nodeRepository.database(); - this.maxActiveUpgrades = maxActiveUpgrades; - this.softRebuildFlag = Flags.SOFT_REBUILD.bindTo(nodeRepository.flagSource()); this.cloud = Objects.requireNonNull(cloud); // Read and write all versions to make sure they are stored in the latest version of the serialized format @@ -141,12 +132,13 @@ public class OsVersions { /** Returns the upgrader to use when upgrading given node type to target */ private OsUpgrader chooseUpgrader(NodeType nodeType, Optional<Version> target) { if (cloud.dynamicProvisioning()) { - boolean canSoftRebuild = cloud.name().equals(CloudName.AWS) && softRebuildFlag.value(); - RetiringOsUpgrader retiringOsUpgrader = new RetiringOsUpgrader(nodeRepository, canSoftRebuild, maxActiveUpgrades); + boolean canSoftRebuild = cloud.name().equals(CloudName.AWS); + RetiringOsUpgrader retiringOsUpgrader = new RetiringOsUpgrader(nodeRepository, canSoftRebuild); if (canSoftRebuild) { // If soft rebuild is enabled, we can use RebuildingOsUpgrader for hosts with remote storage. // RetiringOsUpgrader is then only used for hosts with local storage. - return new CompositeOsUpgrader(List.of(new RebuildingOsUpgrader(nodeRepository, canSoftRebuild), + return new CompositeOsUpgrader(nodeRepository, + List.of(new RebuildingOsUpgrader(nodeRepository, canSoftRebuild), retiringOsUpgrader)); } return retiringOsUpgrader; @@ -161,7 +153,7 @@ public class OsVersions { if (rebuildRequired) { return new RebuildingOsUpgrader(nodeRepository, false); } - return new DelegatingOsUpgrader(nodeRepository, maxActiveUpgrades); + return new DelegatingOsUpgrader(nodeRepository); } private static void requireNonEmpty(Version version) { diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java index dccc22deced..f329c4cb695 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java @@ -3,8 +3,6 @@ package com.yahoo.vespa.hosted.provision.os; import com.yahoo.component.Version; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; import com.yahoo.vespa.hosted.provision.NodeRepository; @@ -31,17 +29,14 @@ import java.util.logging.Logger; * * @author mpolden */ -public class RebuildingOsUpgrader implements OsUpgrader { +public class RebuildingOsUpgrader extends OsUpgrader { private static final Logger LOG = Logger.getLogger(RebuildingOsUpgrader.class.getName()); - private final NodeRepository nodeRepository; - private final IntFlag maxRebuilds; private final boolean softRebuild; public RebuildingOsUpgrader(NodeRepository nodeRepository, boolean softRebuild) { - this.nodeRepository = nodeRepository; - this.maxRebuilds = PermanentFlags.MAX_REBUILDS.bindTo(nodeRepository.flagSource()); + super(nodeRepository); this.softRebuild = softRebuild; } @@ -57,20 +52,12 @@ public class RebuildingOsUpgrader implements OsUpgrader { // No action needed in this implementation. Hosts that have started rebuilding cannot be halted } - /** Returns the number of hosts of given type that can be rebuilt concurrently */ - private int rebuildLimit(NodeType hostType, NodeList hostsOfType) { - if (hostsOfType.stream().anyMatch(host -> host.type() != hostType)) illegal("All hosts must be a " + hostType); - int limit = hostType == NodeType.host ? maxRebuilds.value() : 1; - return Math.max(0, limit - hostsOfType.rebuilding(softRebuild).size()); - } - private List<Node> rebuildableHosts(OsVersionTarget target, NodeList allNodes, Instant now) { NodeList hostsOfTargetType = allNodes.nodeType(target.nodeType()); if (softRebuild) { // Soft rebuild is enabled so this should act on hosts having replacable root disk hostsOfTargetType = hostsOfTargetType.replaceableRootDisk(); } - int rebuildLimit = rebuildLimit(target.nodeType(), hostsOfTargetType); // Find stateful clusters with retiring nodes NodeList activeNodes = allNodes.state(Node.State.active); @@ -79,12 +66,13 @@ public class RebuildingOsUpgrader implements OsUpgrader { .statefulClusters()); // Rebuild hosts not containing stateful clusters with retiring nodes, up to rebuild limit + NodeList activeHosts = hostsOfTargetType.state(Node.State.active); + int rebuildLimit = upgradeSlots(target, activeHosts); List<Node> hostsToRebuild = new ArrayList<>(rebuildLimit); - NodeList candidates = hostsOfTargetType.state(Node.State.active) - .not().rebuilding(softRebuild) - .osVersionIsBefore(target.version()) - .matching(node -> canUpgradeAt(now, node)) - .byIncreasingOsVersion(); + NodeList candidates = activeHosts.not().rebuilding(softRebuild) + .osVersionIsBefore(target.version()) + .matching(node -> canUpgradeAt(now, node)) + .byIncreasingOsVersion(); for (Node host : candidates) { if (hostsToRebuild.size() == rebuildLimit) break; Set<ClusterId> clustersOnHost = activeNodes.childrenOf(host).statefulClusters(); diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java index 9dcf95be477..de0ef085bc6 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java @@ -21,21 +21,15 @@ import java.util.logging.Logger; * * @author mpolden */ -public class RetiringOsUpgrader implements OsUpgrader { +public class RetiringOsUpgrader extends OsUpgrader { private static final Logger LOG = Logger.getLogger(RetiringOsUpgrader.class.getName()); - protected final NodeRepository nodeRepository; - private final boolean softRebuild; - private final int maxActiveUpgrades; - public RetiringOsUpgrader(NodeRepository nodeRepository, boolean softRebuild, int maxActiveUpgrades) { - this.nodeRepository = nodeRepository; + public RetiringOsUpgrader(NodeRepository nodeRepository, boolean softRebuild) { + super(nodeRepository); this.softRebuild = softRebuild; - this.maxActiveUpgrades = maxActiveUpgrades; - if (maxActiveUpgrades < 1) throw new IllegalArgumentException("maxActiveUpgrades must be positive, was " + - maxActiveUpgrades); } @Override @@ -64,14 +58,11 @@ public class RetiringOsUpgrader implements OsUpgrader { // Retire only hosts which do not have a replaceable root disk activeNodes = activeNodes.not().replaceableRootDisk(); } - if (activeNodes.isEmpty()) return NodeList.of(); - - int numberToDeprovision = Math.max(0, maxActiveUpgrades - activeNodes.deprovisioning().size()); return activeNodes.not().deprovisioning() .osVersionIsBefore(target.version()) .matching(node -> canUpgradeAt(instant, node)) .byIncreasingOsVersion() - .first(numberToDeprovision); + .first(upgradeSlots(target, activeNodes)); } /** Upgrade given host by retiring and deprovisioning it */ diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java index 83b4e70c6f0..f9cc41cd8d0 100644 --- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java +++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java @@ -10,7 +10,6 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostSpec; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.NodeType; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.provision.Node; import com.yahoo.vespa.hosted.provision.NodeList; @@ -97,7 +96,8 @@ public class OsVersionsTest { public void max_active_upgrades() { int totalNodes = 20; int maxActiveUpgrades = 5; - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), maxActiveUpgrades); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + setMaxActiveUpgrades(maxActiveUpgrades); provisionInfraApplication(totalNodes); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().state(Node.State.active).hosts(); @@ -163,7 +163,8 @@ public class OsVersionsTest { @Test public void upgrade_by_retiring() { int maxActiveUpgrades = 2; - var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), maxActiveUpgrades); + var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build()); + setMaxActiveUpgrades(maxActiveUpgrades); int hostCount = 10; // Provision hosts and children List<Node> hosts = provisionInfraApplication(hostCount); @@ -211,32 +212,33 @@ public class OsVersionsTest { @Test public void upgrade_by_retiring_everything_at_once() { - var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Integer.MAX_VALUE); + var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build()); + setMaxActiveUpgrades(Integer.MAX_VALUE); int hostCount = 3; - provisionInfraApplication(hostCount, infraApplication, NodeType.confighost); + provisionInfraApplication(hostCount, infraApplication, NodeType.host); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list() - .nodeType(NodeType.confighost) + .nodeType(NodeType.host) .not().state(Node.State.deprovisioned); tester.clock().advance(Duration.ofDays(2)); // Let grace period pass // Target is set and upgrade started var version1 = Version.fromString("7.1"); - versions.setTarget(NodeType.confighost, version1, false); + versions.setTarget(NodeType.host, version1, false); for (int i = 0; i < hostCount; i++) { - versions.resumeUpgradeOf(NodeType.confighost, true); + versions.resumeUpgradeOf(NodeType.host, true); } // All hosts are deprovisioning assertEquals(hostCount, hostNodes.get().deprovisioning().size()); // Nodes complete their upgrade by being reprovisioned - completeReprovisionOf(hostNodes.get().deprovisioning().asList(), NodeType.confighost); + completeReprovisionOf(hostNodes.get().deprovisioning().asList(), NodeType.host); assertEquals(hostCount, hostNodes.get().onOsVersion(version1).size()); } @Test public void upgrade_by_rebuilding() { - tester.flagSource().withIntFlag(PermanentFlags.MAX_REBUILDS.id(), 1); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Integer.MAX_VALUE); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); + setMaxActiveUpgrades(1); int hostCount = 10; provisionInfraApplication(hostCount + 1); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); @@ -287,6 +289,7 @@ public class OsVersionsTest { assertEquals(0, hostNodes.get().rebuilding(false).size()); // Next version is within same major. Upgrade mechanism switches to delegated + setMaxActiveUpgrades(100); var version2 = Version.fromString("8.1"); versions.setTarget(NodeType.host, version2, false); versions.resumeUpgradeOf(NodeType.host, true); @@ -313,13 +316,12 @@ public class OsVersionsTest { int hostCount = 12; boolean softRebuild = true; - tester.flagSource().withIntFlag(PermanentFlags.MAX_REBUILDS.id(), maxRebuilds); - tester.flagSource().withBooleanFlag(Flags.SOFT_REBUILD.id(), softRebuild); + setMaxActiveUpgrades(maxRebuilds); var versions = new OsVersions(tester.nodeRepository(), Cloud.builder() .dynamicProvisioning(true) .name(CloudName.AWS) .account(CloudAccount.from("000000000000")) - .build(), Integer.MAX_VALUE); + .build()); provisionInfraApplication(hostCount, infraApplication, NodeType.host, NodeResources.StorageType.remote); Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host); @@ -363,8 +365,8 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_multiple_host_types() { - tester.flagSource().withIntFlag(PermanentFlags.MAX_REBUILDS.id(), 1); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Integer.MAX_VALUE); + setMaxActiveUpgrades(1); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); int hostCount = 3; provisionInfraApplication(hostCount, infraApplication, NodeType.host); provisionInfraApplication(hostCount, ApplicationId.from("hosted-vespa", "confighost", "default"), NodeType.confighost); @@ -396,8 +398,8 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_is_limited_by_stateful_clusters() { - tester.flagSource().withIntFlag(PermanentFlags.MAX_REBUILDS.id(), 3); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Integer.MAX_VALUE); + setMaxActiveUpgrades(3); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); int hostCount = 5; ApplicationId app1 = ApplicationId.from("t1", "a1", "i1"); ApplicationId app2 = ApplicationId.from("t2", "a2", "i2"); @@ -427,7 +429,7 @@ public class OsVersionsTest { for (int i = 0; i < hostsRebuilding.size(); i++) { Optional<ApplicationId> owner = owners.get(i); List<Node> retiringChildren = allNodes.childrenOf(hostsRebuilding.get(i)).retiring().asList(); - assertEquals(owner.isPresent() ? 1 : 0, retiringChildren.size()); + assertEquals("Retiring children of " + hostsRebuilding.get(i) + ": " + retiringChildren, owner.isPresent() ? 1 : 0, retiringChildren.size()); assertEquals("Rebuilding host of " + owner.map(ApplicationId::toString) .orElse("no application"), owner, @@ -474,8 +476,8 @@ public class OsVersionsTest { @Test public void upgrade_by_rebuilding_limits_infrastructure_host() { int hostCount = 3; - tester.flagSource().withIntFlag(PermanentFlags.MAX_REBUILDS.id(), hostCount); - var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Integer.MAX_VALUE); + setMaxActiveUpgrades(hostCount); + var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud()); provisionInfraApplication(hostCount, infraApplication, NodeType.proxyhost); Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.proxyhost); @@ -497,6 +499,10 @@ public class OsVersionsTest { } } + private void setMaxActiveUpgrades(int max) { + tester.flagSource().withIntFlag(PermanentFlags.MAX_OS_UPGRADES.id(), max); + } + private void deployApplication(ApplicationId application) { ClusterSpec contentSpec = ClusterSpec.request(ClusterSpec.Type.content, ClusterSpec.Id.from("content1")).vespaVersion("7").build(); List<HostSpec> hostSpecs = tester.prepare(application, contentSpec, 2, 1, new NodeResources(4, 8, 100, 0.3)); |