Skip upgrade when OS version is unavailable

10 files changed, 135 insertions, 36 deletions

diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
index 52b68708eee..539f3128091 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/NodeRepository.java
@@ -137,7 +137,7 @@ public class NodeRepository extends AbstractComponent implements HealthCheckerPr
         this.resourcesCalculator = provisionServiceProvider.getHostResourcesCalculator();
         this.nodeResourceLimits = new NodeResourceLimits(this);
         this.nameResolver = nameResolver;
-        this.osVersions = new OsVersions(this);
+        this.osVersions = new OsVersions(this, provisionServiceProvider.getHostProvisioner());
         this.infrastructureVersions = new InfrastructureVersions(db);
         this.firmwareChecks = new FirmwareChecks(db, clock);
         this.containerImages = new ContainerImages(containerImage, tenantContainerImage, tenantGpuContainerImage);
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java
index 02f1b951c8e..05831d2c074 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/CompositeOsUpgrader.java
@@ -3,8 +3,10 @@ package com.yahoo.vespa.hosted.provision.os;
 
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 
 import java.util.List;
+import java.util.Optional;
 
 /**
  * An implementation of {@link OsUpgrader} that delegates calls to multiple implementations.
@@ -15,8 +17,8 @@ public class CompositeOsUpgrader extends OsUpgrader {
 
     private final List<OsUpgrader> upgraders;
 
-    public CompositeOsUpgrader(NodeRepository nodeRepository, List<OsUpgrader> upgraders) {
-        super(nodeRepository);
+    public CompositeOsUpgrader(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, List<OsUpgrader> upgraders) {
+        super(nodeRepository, hostProvisioner);
         this.upgraders = List.copyOf(upgraders);
     }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java
index c2d3f511711..7353b70923c 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/DelegatingOsUpgrader.java
@@ -6,6 +6,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 
 import java.time.Instant;
 import java.util.Optional;
@@ -23,8 +24,8 @@ public class DelegatingOsUpgrader extends OsUpgrader {
 
     private static final Logger LOG = Logger.getLogger(DelegatingOsUpgrader.class.getName());
 
-    public DelegatingOsUpgrader(NodeRepository nodeRepository) {
-        super(nodeRepository);
+    public DelegatingOsUpgrader(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) {
+        super(nodeRepository, hostProvisioner);
     }
 
     @Override
@@ -35,7 +36,7 @@ public class DelegatingOsUpgrader extends OsUpgrader {
                                              // This upgrader cannot downgrade nodes. We therefore consider only nodes
                                              // on a lower version than the target
                                              .osVersionIsBefore(target.version())
-                                             .matching(node -> canUpgradeAt(now, node))
+                                             .matching(node -> canUpgradeTo(target.version(), now, node))
                                              .byIncreasingOsVersion()
                                              .first(upgradeSlots(target, activeNodes));
         if (nodesToUpgrade.size() == 0) return;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java
index 436181f99ba..85a46591aa3 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsUpgrader.java
@@ -1,15 +1,28 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.os;
 
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.vespa.flags.IntFlag;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
+import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * Interface for an OS upgrader.
@@ -18,13 +31,23 @@ import java.time.Instant;
  */
 public abstract class OsUpgrader {
 
+    private final Logger LOG = Logger.getLogger(OsUpgrader.class.getName());
+
     private final IntFlag maxActiveUpgrades;
+    private final Optional<HostProvisioner> hostProvisioner;
+    // Supported versions is queried for each host to upgrade, so we cache the results for a while to avoid excessive
+    // API calls to the host provisioner
+    private final Cache<CloudAccount, Set<Version>> supportedVersions = CacheBuilder.newBuilder()
+                                                                                    .expireAfterWrite(10, TimeUnit.MINUTES)
+                                                                                    .build();
 
     final NodeRepository nodeRepository;
 
-    public OsUpgrader(NodeRepository nodeRepository) {
-        this.nodeRepository = nodeRepository;
+
+    public OsUpgrader(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) {
+        this.nodeRepository = Objects.requireNonNull(nodeRepository);
         this.maxActiveUpgrades = PermanentFlags.MAX_OS_UPGRADES.bindTo(nodeRepository.flagSource());
+        this.hostProvisioner = Objects.requireNonNull(hostProvisioner);
     }
 
     /** Trigger upgrade to given target */
@@ -43,10 +66,31 @@ public abstract class OsUpgrader {
         return Math.max(0, max - upgrading);
     }
 
-    /** Returns whether node can change version at given instant */
-    final boolean canUpgradeAt(Instant instant, Node node) {
-        return node.status().osVersion().downgrading() || // Fast-track downgrades
-               node.history().age(instant).compareTo(gracePeriod()) > 0;
+    /** Returns whether node can upgrade to version at given instant */
+    final boolean canUpgradeTo(Version version, Instant instant, Node node) {
+        Set<Version> versions = supportedVersions(node, version);
+        boolean versionAvailable = versions.contains(version);
+        if (!versionAvailable) {
+            LOG.log(Level.WARNING, "Want to upgrade host " + node.hostname() + " to OS version " +
+                                   version.toFullString() + ", but this version does not exist in " +
+                                   node.cloudAccount() + ". Found " + versions.stream().sorted().toList());
+        }
+        return versionAvailable &&
+               (node.status().osVersion().downgrading() || // Fast-track downgrades
+                node.history().age(instant).compareTo(gracePeriod()) > 0);
+    }
+
+    private Set<Version> supportedVersions(Node host, Version requestedVersion) {
+        if (hostProvisioner.isEmpty()) {
+            return Set.of(requestedVersion);
+        }
+        try {
+            return supportedVersions.get(host.cloudAccount(),
+                                         () -> hostProvisioner.get().osVersions(host, requestedVersion.getMajor()));
+        } catch (ExecutionException e) {
+            LOG.log(Level.WARNING, "Failed to list supported OS versions in " + host.cloudAccount() + ": " + Exceptions.toMessageString(e));
+            return Set.of();
+        }
     }
 
     /** The duration this leaves new nodes alone before scheduling any upgrade */
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
index daed86dc2ab..dc7e51caf4e 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/OsVersions.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 
 import java.util.List;
 import java.util.Objects;
@@ -35,15 +36,17 @@ public class OsVersions {
     private final NodeRepository nodeRepository;
     private final CuratorDb db;
     private final Cloud cloud;
+    private final Optional<HostProvisioner> hostProvisioner;
 
-    public OsVersions(NodeRepository nodeRepository) {
-        this(nodeRepository, nodeRepository.zone().cloud());
+    public OsVersions(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner) {
+        this(nodeRepository, nodeRepository.zone().cloud(), hostProvisioner);
     }
 
-    OsVersions(NodeRepository nodeRepository, Cloud cloud) {
+    OsVersions(NodeRepository nodeRepository, Cloud cloud, Optional<HostProvisioner> hostProvisioner) {
         this.nodeRepository = Objects.requireNonNull(nodeRepository);
         this.db = nodeRepository.database();
         this.cloud = Objects.requireNonNull(cloud);
+        this.hostProvisioner = Objects.requireNonNull(hostProvisioner);
 
         // Read and write all versions to make sure they are stored in the latest version of the serialized format
         try (var lock = db.lockOsVersionChange()) {
@@ -126,19 +129,24 @@ public class OsVersions {
 
     /** Returns whether node can be upgraded now */
     public boolean canUpgrade(Node node) {
-        return chooseUpgrader(node.type(), Optional.empty()).canUpgradeAt(nodeRepository.clock().instant(), node);
+        Optional<Version> wantedVersion = node.status().osVersion().wanted();
+        if (wantedVersion.isEmpty()) {
+            return false;
+        }
+        return chooseUpgrader(node.type(), Optional.empty()).canUpgradeTo(wantedVersion.get(), nodeRepository.clock().instant(), node);
     }
 
     /** Returns the upgrader to use when upgrading given node type to target */
     private OsUpgrader chooseUpgrader(NodeType nodeType, Optional<Version> target) {
         if (cloud.dynamicProvisioning()) {
             boolean canSoftRebuild = cloud.name().equals(CloudName.AWS);
-            RetiringOsUpgrader retiringOsUpgrader = new RetiringOsUpgrader(nodeRepository, canSoftRebuild);
+            RetiringOsUpgrader retiringOsUpgrader = new RetiringOsUpgrader(nodeRepository, hostProvisioner, canSoftRebuild);
             if (canSoftRebuild) {
                 // If soft rebuild is enabled, we can use RebuildingOsUpgrader for hosts with remote storage.
                 // RetiringOsUpgrader is then only used for hosts with local storage.
                 return new CompositeOsUpgrader(nodeRepository,
-                                               List.of(new RebuildingOsUpgrader(nodeRepository, canSoftRebuild),
+                                               hostProvisioner,
+                                               List.of(new RebuildingOsUpgrader(nodeRepository, hostProvisioner, canSoftRebuild),
                                                        retiringOsUpgrader));
             }
             return retiringOsUpgrader;
@@ -151,9 +159,9 @@ public class OsVersions {
                                                 .anyMatch(osVersion -> osVersion.current().isPresent() &&
                                                                        osVersion.current().get().getMajor() < target.get().getMajor());
         if (rebuildRequired) {
-            return new RebuildingOsUpgrader(nodeRepository, false);
+            return new RebuildingOsUpgrader(nodeRepository, hostProvisioner, false);
         }
-        return new DelegatingOsUpgrader(nodeRepository);
+        return new DelegatingOsUpgrader(nodeRepository, hostProvisioner);
     }
 
     private static void requireNonEmpty(Version version) {
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java
index 108093d8379..31e6a4e6e26 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RebuildingOsUpgrader.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.ClusterId;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 
 import java.time.Instant;
 import java.util.ArrayList;
@@ -35,8 +36,8 @@ public class RebuildingOsUpgrader extends OsUpgrader {
 
     private final boolean softRebuild;
 
-    public RebuildingOsUpgrader(NodeRepository nodeRepository, boolean softRebuild) {
-        super(nodeRepository);
+    public RebuildingOsUpgrader(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, boolean softRebuild) {
+        super(nodeRepository, hostProvisioner);
         this.softRebuild = softRebuild;
     }
 
@@ -71,7 +72,7 @@ public class RebuildingOsUpgrader extends OsUpgrader {
         List<Node> hostsToRebuild = new ArrayList<>(rebuildLimit);
         NodeList candidates = hosts.not().rebuilding(softRebuild)
                                    .not().onOsVersion(target.version())
-                                   .matching(node -> canUpgradeAt(now, node))
+                                   .matching(node -> canUpgradeTo(target.version(), now, node))
                                    .byIncreasingOsVersion();
         for (Node host : candidates) {
             if (hostsToRebuild.size() == rebuildLimit) break;
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java
index c1e8f2b6fa4..653ff2a61c1 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/os/RetiringOsUpgrader.java
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.provision.NodeList;
 import com.yahoo.vespa.hosted.provision.NodeRepository;
 import com.yahoo.vespa.hosted.provision.node.Agent;
 import com.yahoo.vespa.hosted.provision.node.filter.NodeListFilter;
+import com.yahoo.vespa.hosted.provision.provisioning.HostProvisioner;
 
 import java.time.Instant;
 import java.util.Optional;
@@ -26,8 +27,8 @@ public class RetiringOsUpgrader extends OsUpgrader {
 
     private final boolean softRebuild;
 
-    public RetiringOsUpgrader(NodeRepository nodeRepository, boolean softRebuild) {
-        super(nodeRepository);
+    public RetiringOsUpgrader(NodeRepository nodeRepository, Optional<HostProvisioner> hostProvisioner, boolean softRebuild) {
+        super(nodeRepository, hostProvisioner);
         this.softRebuild = softRebuild;
     }
 
@@ -54,7 +55,7 @@ public class RetiringOsUpgrader extends OsUpgrader {
         }
         return nodes.not().deprovisioning()
                     .not().onOsVersion(target.version())
-                    .matching(node -> canUpgradeAt(instant, node))
+                    .matching(node -> canUpgradeTo(target.version(), instant, node))
                     .byIncreasingOsVersion()
                     .first(upgradeSlots(target, nodes.deprovisioning()));
     }
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
index 8ef4b6c8bd1..4214f543c60 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/provisioning/HostProvisioner.java
@@ -1,6 +1,7 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.provisioning;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.HostEvent;
@@ -11,6 +12,7 @@ import com.yahoo.vespa.hosted.provision.Node;
 import java.util.Collection;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
 
@@ -94,6 +96,9 @@ public interface HostProvisioner {
     /** Returns whether flavor for given host can be upgraded to a newer generation */
     boolean canUpgradeFlavor(Node host, Node child, Predicate<NodeResources> realHostResourcesWithinLimits);
 
+    /** Returns all OS versions available to host for the given major version */
+    Set<Version> osVersions(Node host, int majorVersion);
+
     /** Updates the given hosts to indicate that they are allocated to the given application. */
     default void updateAllocation(Collection<Node> hosts, ApplicationId owner) { }
 
diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
index b5bb91af71a..73985075319 100644
--- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
+++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/testutils/MockHostProvisioner.java
@@ -1,6 +1,7 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.provision.testutils;
 
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Flavor;
@@ -32,6 +33,7 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.function.Consumer;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
 import static com.yahoo.config.provision.NodeType.host;
@@ -50,6 +52,7 @@ public class MockHostProvisioner implements HostProvisioner {
     private final Map<ClusterSpec.Type, Flavor> hostFlavors = new HashMap<>();
     private final Set<String> upgradableFlavors = new HashSet<>();
     private final Map<Behaviour, Integer> behaviours = new HashMap<>();
+    private final Set<Version> osVersions = new HashSet<>();
 
     private int deprovisionedHosts = 0;
 
@@ -146,6 +149,11 @@ public class MockHostProvisioner implements HostProvisioner {
         return upgradableFlavors.contains(host.flavor().name());
     }
 
+    @Override
+    public Set<Version> osVersions(Node host, int majorVersion) {
+        return osVersions.stream().filter(v -> v.getMajor() == majorVersion).collect(Collectors.toUnmodifiableSet());
+    }
+
     /** Returns the hosts that have been provisioned by this  */
     public List<ProvisionedHost> provisionedHosts() {
         return Collections.unmodifiableList(provisionedHosts);
@@ -214,6 +222,11 @@ public class MockHostProvisioner implements HostProvisioner {
         return this;
     }
 
+    public MockHostProvisioner addOsVersion(Version version) {
+        osVersions.add(version);
+        return this;
+    }
+
     public boolean compatible(Flavor flavor, NodeResources resources) {
         NodeResources resourcesToVerify = resources.withMemoryGb(resources.memoryGb() - memoryTaxGb);
 
diff --git a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
index 0be90dcb888..b7014fbd280 100644
--- a/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
+++ b/node-repository/src/test/java/com/yahoo/vespa/hosted/provision/os/OsVersionsTest.java
@@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.provision.node.Allocation;
 import com.yahoo.vespa.hosted.provision.node.OsVersion;
 import com.yahoo.vespa.hosted.provision.node.Status;
 import com.yahoo.vespa.hosted.provision.provisioning.ProvisioningTester;
+import com.yahoo.vespa.hosted.provision.testutils.MockHostProvisioner;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -41,7 +42,7 @@ public class OsVersionsTest {
 
     @Test
     public void upgrade() {
-        var versions = new OsVersions(tester.nodeRepository());
+        var versions = new OsVersions(tester.nodeRepository(), Optional.ofNullable(tester.hostProvisioner()));
         provisionInfraApplication(10);
         Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host);
 
@@ -94,7 +95,7 @@ public class OsVersionsTest {
     public void max_active_upgrades() {
         int totalNodes = 20;
         int maxActiveUpgrades = 5;
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner()));
         setMaxActiveUpgrades(maxActiveUpgrades);
         provisionInfraApplication(totalNodes);
         Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().state(Node.State.active).hosts();
@@ -140,7 +141,7 @@ public class OsVersionsTest {
 
     @Test
     public void newer_upgrade_aborts_upgrade_to_stale_version() {
-        var versions = new OsVersions(tester.nodeRepository());
+        var versions = new OsVersions(tester.nodeRepository(), Optional.ofNullable(tester.hostProvisioner()));
         provisionInfraApplication(10);
         Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().hosts();
 
@@ -160,7 +161,7 @@ public class OsVersionsTest {
     @Test
     public void upgrade_and_downgrade_by_retiring() {
         int maxActiveUpgrades = 2;
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.ofNullable(tester.hostProvisioner()));
         setMaxActiveUpgrades(maxActiveUpgrades);
         int hostCount = 10;
         // Provision hosts and children
@@ -229,7 +230,7 @@ public class OsVersionsTest {
 
     @Test
     public void upgrade_by_retiring_everything_at_once() {
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.ofNullable(tester.hostProvisioner()));
         setMaxActiveUpgrades(Integer.MAX_VALUE);
         int hostCount = 3;
         provisionInfraApplication(hostCount, NodeType.host);
@@ -253,7 +254,7 @@ public class OsVersionsTest {
 
     @Test
     public void upgrade_by_rebuilding() {
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner()));
         setMaxActiveUpgrades(1);
         int hostCount = 10;
         provisionInfraApplication(hostCount + 1);
@@ -337,7 +338,8 @@ public class OsVersionsTest {
                                                                     .dynamicProvisioning(true)
                                                                     .name(CloudName.AWS)
                                                                     .account(CloudAccount.from("000000000000"))
-                                                                    .build());
+                                                                    .build(),
+                                      Optional.ofNullable(tester.hostProvisioner()));
 
         provisionInfraApplication(hostCount, NodeType.host, NodeResources.StorageType.remote, NodeResources.Architecture.x86_64);
         Supplier<NodeList> hostNodes = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host);
@@ -382,7 +384,7 @@ public class OsVersionsTest {
     @Test
     public void upgrade_by_rebuilding_multiple_host_types() {
         setMaxActiveUpgrades(1);
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner()));
         int hostCount = 3;
         provisionInfraApplication(hostCount, NodeType.host);
         provisionInfraApplication(hostCount, NodeType.confighost);
@@ -415,7 +417,7 @@ public class OsVersionsTest {
     @Test
     public void upgrade_by_rebuilding_is_limited_by_stateful_clusters() {
         setMaxActiveUpgrades(3);
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner()));
         int hostCount = 5;
         ApplicationId app1 = ApplicationId.from("t1", "a1", "i1");
         ApplicationId app2 = ApplicationId.from("t2", "a2", "i2");
@@ -493,7 +495,7 @@ public class OsVersionsTest {
     public void upgrade_by_rebuilding_limits_infrastructure_host() {
         int hostCount = 3;
         setMaxActiveUpgrades(hostCount);
-        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud());
+        var versions = new OsVersions(tester.nodeRepository(), Cloud.defaultCloud(), Optional.ofNullable(tester.hostProvisioner()));
         provisionInfraApplication(hostCount, NodeType.proxyhost);
         Supplier<NodeList> hosts = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.proxyhost);
 
@@ -515,6 +517,28 @@ public class OsVersionsTest {
         }
     }
 
+    @Test
+    public void skips_unavailable_version() {
+        MockHostProvisioner hostProvisioner = new MockHostProvisioner(List.of());
+        ProvisioningTester tester = new ProvisioningTester.Builder().dynamicProvisioning(true, false).hostProvisioner(hostProvisioner).build();
+        OsVersions versions = new OsVersions(tester.nodeRepository(), Cloud.builder().dynamicProvisioning(true).build(), Optional.of(hostProvisioner));
+        tester.makeReadyHosts(1, new NodeResources(2,4,8,100));
+        tester.activateTenantHosts();
+        Supplier<Node> host = () -> tester.nodeRepository().nodes().list().nodeType(NodeType.host).first().get();
+        tester.clock().advance(Duration.ofDays(1));
+
+        hostProvisioner.addOsVersion(Version.fromString("7.0"));
+        Version version0 = Version.fromString("8.0");
+        versions.setTarget(NodeType.host, version0, false);
+        versions.resumeUpgradeOf(NodeType.host, true);
+        assertTrue("Upgrade is not triggered to unavailable version", host.get().status().osVersion().wanted().isEmpty());
+
+        // Version becomes available
+        hostProvisioner.addOsVersion(version0);
+        versions.resumeUpgradeOf(NodeType.host, true);
+        assertEquals("Host upgrade is triggered", version0, host.get().status().osVersion().wanted().get());
+    }
+
     private void setMaxActiveUpgrades(int max) {
         tester.flagSource().withIntFlag(PermanentFlags.MAX_OS_UPGRADES.id(), max);
     }