Merge pull request #20629 from vespa-engine/mpolden/downgrade

Support aborting and rolling back incomplete upgrade

24 files changed, 301 insertions, 94 deletions

diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java
index 91ecf7b29e4..85cc384660d 100644
--- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java
@@ -2,45 +2,68 @@
 package com.yahoo.config.provision.zone;
 
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
 
 /**
- * This class declares the order to use when upgrading zones in a system.
+ * This class declares the steps (zones) to follow when upgrading a system. If a step contains multiple zones, those
+ * will be upgraded in parallel.
  *
  * @author mpolden
  */
 public class UpgradePolicy {
 
-    private final List<List<ZoneApi>> zones;
+    private final List<Set<ZoneApi>> steps;
 
-    private UpgradePolicy(List<List<ZoneApi>> zones) {
-        this.zones = zones;
+    private UpgradePolicy(List<Set<ZoneApi>> steps) {
+        for (int i = 0; i < steps.size(); i++) {
+            for (int j = 0; j < i; j++) {
+                if (!Collections.disjoint(steps.get(i), steps.get(j))) {
+                    throw new IllegalArgumentException("One or more zones are declared in multiple steps");
+                }
+            }
+        }
+        this.steps = List.copyOf(steps);
     }
 
-    public List<List<ZoneApi>> asList() {
-        return List.copyOf(zones);
+    /** Returns the steps in this */
+    public List<Set<ZoneApi>> steps() {
+        return steps;
     }
 
-    private UpgradePolicy with(ZoneApi... zone) {
-        List<List<ZoneApi>> zones = new ArrayList<>(this.zones);
-        zones.add(Arrays.asList(zone));
-        return new UpgradePolicy(zones);
+    /** Returns a copy of this with the step order inverted */
+    public UpgradePolicy inverted() {
+        List<Set<ZoneApi>> copy = new ArrayList<>(steps);
+        Collections.reverse(copy);
+        return new UpgradePolicy(copy);
     }
 
-    /** Upgrade given zone as the next step */
-    public UpgradePolicy upgrade(ZoneApi zone) {
-        return with(zone);
+    public static UpgradePolicy.Builder builder() {
+        return new UpgradePolicy.Builder();
     }
 
-    /** Upgrade given zones in parallel as the next step */
-    public UpgradePolicy upgradeInParallel(ZoneApi... zone) {
-        return with(zone);
-    }
+    public static class Builder {
+
+        private final List<Set<ZoneApi>> steps = new ArrayList<>();
+
+        private Builder() {}
+
+        /** Upgrade given zone as the next step */
+        public Builder upgrade(ZoneApi zone) {
+            return upgradeInParallel(zone);
+        }
+
+        /** Upgrade given zones in parallel as the next step */
+        public Builder upgradeInParallel(ZoneApi... zone) {
+            this.steps.add(Set.of(zone));
+            return this;
+        }
+
+        public UpgradePolicy build() {
+            return new UpgradePolicy(steps);
+        }
 
-    public static UpgradePolicy create() {
-        return new UpgradePolicy(Collections.emptyList());
     }
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
index 0208f9ecd0d..395f0642965 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java
@@ -55,7 +55,7 @@ public interface NodeRepository {
     void removeArchiveUri(ZoneId zone, TenantName tenantName);
 
     /** Upgrade all nodes of given type to a new version */
-    void upgrade(ZoneId zone, NodeType type, Version version);
+    void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade);
 
     /** Upgrade OS for all nodes of given type to a new version */
     void upgradeOs(ZoneId zone, NodeType type, Version version, Duration upgradeBudget);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java
index 20dd0a8739d..f1fe2524e99 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/SystemMonitor.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.organization;
 import com.yahoo.component.Version;
 
 /**
- * Montitors a Vespa controller and its system.
+ * Monitors a Vespa controller and its system.
  *
  * @author jonmv
  */
@@ -14,7 +14,7 @@ public interface SystemMonitor {
     void reportSystemVersion(Version systemVersion, Confidence confidence);
 
     enum Confidence {
-        broken, low, normal, high;
+        aborted, broken, low, normal, high;
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 39223d6c031..81f8831a8a7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -449,12 +449,12 @@ public class ApplicationController {
     }
 
     /** Deploy a system application to given zone */
-    public void deploy(SystemApplication application, ZoneId zone, Version version) {
+    public void deploy(SystemApplication application, ZoneId zone, Version version, boolean allowDowngrade) {
         if (application.hasApplicationPackage()) {
             deploySystemApplicationPackage(application, zone, version);
         } else {
             // Deploy by calling node repository directly
-            configServer.nodeRepository().upgrade(zone, application.nodeType(), version);
+            configServer.nodeRepository().upgrade(zone, application.nodeType(), version, allowDowngrade);
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
index 1d71fa66329..045960fdce2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ArchiveUriUpdater.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
@@ -31,7 +30,7 @@ public class ArchiveUriUpdater extends ControllerMaintainer {
     private final CuratorArchiveBucketDb archiveBucketDb;
 
     public ArchiveUriUpdater(Controller controller, Duration duration) {
-        super(controller, duration, ArchiveUriUpdater.class.getSimpleName(), SystemName.all());
+        super(controller, duration);
         this.applications = controller.applications();
         this.nodeRepository = controller.serviceRegistry().configServer().nodeRepository();
         this.archiveBucketDb = controller.archiveBucketDb();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
index aca154a4b5b..f196ca610c8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java
@@ -1,7 +1,6 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.text.Text;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -40,7 +39,7 @@ public class DeploymentMetricsMaintainer extends ControllerMaintainer {
     private final ApplicationController applications;
 
     public DeploymentMetricsMaintainer(Controller controller, Duration duration) {
-        super(controller, duration, DeploymentMetricsMaintainer.class.getSimpleName(), SystemName.all());
+        super(controller, duration);
         this.applications = controller.applications();
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
index f7e46aaa34a..e96afd0ba8a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.google.common.collect.Sets;
 import com.google.inject.Inject;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.log.LogLevel;
@@ -54,7 +53,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
 
     @Inject
     public EndpointCertificateMaintainer(Controller controller, Duration interval) {
-        super(controller, interval, null, SystemName.all());
+        super(controller, interval);
         this.deploymentTrigger = controller.applications().deploymentTrigger();
         this.clock = controller.clock();
         this.secretStore = controller.secretStore();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
index bb10410d2ef..82413f21222 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
@@ -10,14 +10,19 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.versions.VersionTarget;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
 import java.util.Comparator;
 import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.logging.Logger;
 
@@ -26,7 +31,7 @@ import java.util.logging.Logger;
  *
  * @author mpolden
  */
-public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintainer {
+public abstract class InfrastructureUpgrader<TARGET extends VersionTarget> extends ControllerMaintainer {
 
     private static final Logger log = Logger.getLogger(InfrastructureUpgrader.class.getName());
 
@@ -42,17 +47,19 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain
 
     @Override
     protected double maintain() {
-        if (targetVersion().isEmpty())  return 1.0;
-        return upgradeAll(targetVersion().get(), managedApplications);
+        return target().map(target -> upgradeAll(target, managedApplications))
+                       .orElse(1.0);
     }
 
     /** Deploy a list of system applications until they converge on the given version */
-    private double upgradeAll(VERSION target, List<SystemApplication> applications) {
+    private double upgradeAll(TARGET target, List<SystemApplication> applications) {
         int attempts = 0;
         int failures = 0;
-        for (List<ZoneApi> zones : upgradePolicy.asList()) {
+        // Invert zone order if we're downgrading
+        UpgradePolicy policy = target.downgrade() ? upgradePolicy.inverted() : upgradePolicy;
+        for (Set<ZoneApi> step : policy.steps()) {
             boolean converged = true;
-            for (ZoneApi zone : zones) {
+            for (ZoneApi zone : step) {
                 try {
                     attempts++;
                     converged &= upgradeAll(target, applications, zone);
@@ -76,10 +83,24 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain
     }
 
     /** Returns whether all applications have converged to the target version in zone */
-    private boolean upgradeAll(VERSION target, List<SystemApplication> applications, ZoneApi zone) {
+    private boolean upgradeAll(TARGET target, List<SystemApplication> applications, ZoneApi zone) {
+        Map<SystemApplication, Set<SystemApplication>> dependenciesByApplication = new HashMap<>();
+        if (target.downgrade()) { // Invert dependencies when we're downgrading
+            for (var application : applications) {
+                dependenciesByApplication.computeIfAbsent(application, k -> new HashSet<>());
+                for (var dependency : application.dependencies()) {
+                    dependenciesByApplication.computeIfAbsent(dependency, k -> new HashSet<>())
+                                             .add(application);
+                }
+            }
+        } else {
+            applications.forEach(app -> dependenciesByApplication.put(app, Set.copyOf(app.dependencies())));
+        }
         boolean converged = true;
-        for (SystemApplication application : applications) {
-            if (convergedOn(target, application.dependencies(), zone)) {
+        for (var kv : dependenciesByApplication.entrySet()) {
+            SystemApplication application = kv.getKey();
+            Set<SystemApplication> dependencies = kv.getValue();
+            if (convergedOn(target, dependencies, zone)) {
                 if (changeTargetTo(target, application, zone)) {
                     upgrade(target, application, zone);
                 }
@@ -89,21 +110,21 @@ public abstract class InfrastructureUpgrader<VERSION> extends ControllerMaintain
         return converged;
     }
 
-    private boolean convergedOn(VERSION target, List<SystemApplication> applications, ZoneApi zone) {
+    private boolean convergedOn(TARGET target, Set<SystemApplication> applications, ZoneApi zone) {
         return applications.stream().allMatch(application -> convergedOn(target, application, zone));
     }
 
     /** Returns whether target version for application in zone should be changed */
-    protected abstract boolean changeTargetTo(VERSION target, SystemApplication application, ZoneApi zone);
+    protected abstract boolean changeTargetTo(TARGET target, SystemApplication application, ZoneApi zone);
 
     /** Upgrade component to target version. Implementation should be idempotent */
-    protected abstract void upgrade(VERSION target, SystemApplication application, ZoneApi zone);
+    protected abstract void upgrade(TARGET target, SystemApplication application, ZoneApi zone);
 
     /** Returns whether application has converged to target version in zone */
-    protected abstract boolean convergedOn(VERSION target, SystemApplication application, ZoneApi zone);
+    protected abstract boolean convergedOn(TARGET target, SystemApplication application, ZoneApi zone);
 
-    /** Returns the target version for the component upgraded by this, if any */
-    protected abstract Optional<VERSION> targetVersion();
+    /** Returns the version target for the component upgraded by this, if any */
+    protected abstract Optional<TARGET> target();
 
     /** Returns whether the upgrader should expect given node to upgrade */
     protected abstract boolean expectUpgradeOf(Node node, SystemApplication application, ZoneApi zone);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
index a1e956737cc..fa64a2677f4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
@@ -66,7 +66,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> {
     }
 
     @Override
-    protected Optional<OsVersionTarget> targetVersion() {
+    protected Optional<OsVersionTarget> target() {
         // Return target if we have nodes in this cloud on a lower version
         return controller().osVersionTarget(cloud)
                            .filter(target -> controller().osVersionStatus().nodesIn(cloud).stream()
@@ -90,7 +90,7 @@ public class OsUpgrader extends InfrastructureUpgrader<OsVersionTarget> {
     /** Returns the available upgrade budget for given zone */
     private Duration zoneBudgetOf(Duration totalBudget, ZoneApi zone) {
         if (!spendBudgetOn(zone)) return Duration.ZERO;
-        long consecutiveZones = upgradePolicy.asList().stream()
+        long consecutiveZones = upgradePolicy.steps().stream()
                                              .filter(parallelZones -> parallelZones.stream().anyMatch(this::spendBudgetOn))
                                              .count();
         return totalBudget.dividedBy(consecutiveZones);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
index 8b0371e2c1a..99ab6d420cb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
@@ -8,7 +8,9 @@ import com.yahoo.text.Text;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersionTarget;
 
 import java.time.Duration;
 import java.util.Optional;
@@ -20,7 +22,7 @@ import java.util.logging.Logger;
  *
  * @author mpolden
  */
-public class SystemUpgrader extends InfrastructureUpgrader<Version> {
+public class SystemUpgrader extends InfrastructureUpgrader<VespaVersionTarget> {
 
     private static final Logger log = Logger.getLogger(SystemUpgrader.class.getName());
 
@@ -31,19 +33,19 @@ public class SystemUpgrader extends InfrastructureUpgrader<Version> {
     }
 
     @Override
-    protected void upgrade(Version target, SystemApplication application, ZoneApi zone) {
+    protected void upgrade(VespaVersionTarget target, SystemApplication application, ZoneApi zone) {
         log.info(Text.format("Deploying %s version %s in %s", application.id(), target, zone.getId()));
-        controller().applications().deploy(application, zone.getId(), target);
+        controller().applications().deploy(application, zone.getId(), target.version(), target.downgrade());
     }
 
     @Override
-    protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) {
+    protected boolean convergedOn(VespaVersionTarget target, SystemApplication application, ZoneApi zone) {
         Optional<Version> minVersion = minVersion(zone, application, Node::currentVersion);
         // Skip application convergence check if there are no nodes belonging to the application in the zone
         if (minVersion.isEmpty()) return true;
 
-        return minVersion.get().equals(target) &&
-               application.configConvergedIn(zone.getId(), controller(), Optional.of(target));
+        return minVersion.get().equals(target.version()) &&
+               application.configConvergedIn(zone.getId(), controller(), Optional.of(target.version()));
     }
 
     @Override
@@ -52,30 +54,41 @@ public class SystemUpgrader extends InfrastructureUpgrader<Version> {
     }
 
     @Override
-    protected Optional<Version> targetVersion() {
-        return controller().readVersionStatus().controllerVersion()
-                           .filter(vespaVersion -> !vespaVersion.isSystemVersion())
-                           .filter(vespaVersion -> vespaVersion.confidence() != VespaVersion.Confidence.broken)
-                           .map(VespaVersion::versionNumber);
+    protected Optional<VespaVersionTarget> target() {
+        VersionStatus status = controller().readVersionStatus();
+        Optional<VespaVersion> target = status.controllerVersion()
+                                              .filter(version -> {
+                                                  Version systemVersion = status.systemVersion()
+                                                                                .map(VespaVersion::versionNumber)
+                                                                                .orElse(Version.emptyVersion);
+                                                  return version.versionNumber().isAfter(systemVersion);
+                                              })
+                                              .filter(version -> version.confidence() != VespaVersion.Confidence.broken);
+        boolean downgrade = target.isPresent() && target.get().confidence() == VespaVersion.Confidence.aborted;
+        if (downgrade) {
+            target = status.systemVersion();
+        }
+        return target.map(VespaVersion::versionNumber)
+                     .map(version -> new VespaVersionTarget(version, downgrade));
     }
 
     @Override
-    protected boolean changeTargetTo(Version target, SystemApplication application, ZoneApi zone) {
+    protected boolean changeTargetTo(VespaVersionTarget target, SystemApplication application, ZoneApi zone) {
         if (application.hasApplicationPackage()) {
             // For applications with package we do not have a zone-wide version target. This means that we must check
             // the wanted version of each node.
             boolean zoneHasSharedRouting = controller().zoneRegistry().routingMethods(zone.getId()).stream()
                                                        .anyMatch(RoutingMethod::isShared);
             return minVersion(zone, application, Node::wantedVersion)
-                    .map(target::isAfter)          // Upgrade if target is after any wanted version
+                    .map(wantedVersion -> !wantedVersion.equals(target.version()))
                     .orElse(zoneHasSharedRouting); // Always upgrade if zone uses shared routing, but has no nodes allocated yet
 
         }
         return controller().serviceRegistry().configServer().nodeRepository()
                            .targetVersionsOf(zone.getId())
                            .vespaVersion(application.nodeType())
-                           .map(target::isAfter)                              // Upgrade if target is after current
-                           .orElse(true);                                     // Upgrade if target is unset
+                           .map(wantedVersion -> !wantedVersion.equals(target.version()))
+                           .orElse(true); // Always set target if there are no nodes
     }
 
     /** Returns whether node in application should be upgraded by this */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index be72ac2f09d..7a7eee7183b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -174,6 +174,11 @@ public class Upgrader extends ControllerMaintainer {
 
     /** Override confidence for given version. This will cause the computed confidence to be ignored */
     public void overrideConfidence(Version version, Confidence confidence) {
+        if (confidence == Confidence.aborted && !version.isAfter(controller().readSystemVersion())) {
+            throw new IllegalArgumentException("Cannot override confidence to " + confidence +
+                                               " for version " + version.toFullString() +
+                                               ": Version may be in use by applications");
+        }
         try (Lock lock = curator.lockConfidenceOverrides()) {
             Map<Version, Confidence> overrides = new LinkedHashMap<>(curator.readConfidenceOverrides());
             overrides.put(version, confidence);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
index 6bf73c45965..6597d59027c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/VersionStatusUpdater.java
@@ -14,6 +14,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.organization.Sys
 import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.high;
 import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.low;
 import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.normal;
+import static com.yahoo.vespa.hosted.controller.api.integration.organization.SystemMonitor.Confidence.aborted;
 
 /**
  * This maintenance job periodically updates the version status.
@@ -47,10 +48,11 @@ public class VersionStatusUpdater extends ControllerMaintainer {
 
     static SystemMonitor.Confidence convert(VespaVersion.Confidence confidence) {
         switch (confidence) {
-            case broken: return broken;
-            case low:    return low;
-            case normal: return normal;
-            case high:   return high;
+            case aborted: return aborted;
+            case broken:  return broken;
+            case low:     return low;
+            case normal:  return normal;
+            case high:    return high;
             default: throw new IllegalArgumentException("Unexpected confidence '" + confidence + "'");
         }
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
index c70d9fd20cb..fc7fbe45767 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
@@ -89,7 +89,7 @@ public class OsVersionStatus {
 
     private static List<ZoneApi> zonesToUpgrade(Controller controller) {
         return controller.zoneRegistry().osUpgradePolicies().stream()
-                         .flatMap(upgradePolicy -> upgradePolicy.asList().stream())
+                         .flatMap(upgradePolicy -> upgradePolicy.steps().stream())
                          .flatMap(Collection::stream)
                          .collect(Collectors.toUnmodifiableList());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java
index a8fcb6c78fc..1c27058a6ef 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionTarget.java
@@ -1,16 +1,18 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
+import com.yahoo.component.Version;
+
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Objects;
 
 /**
- * An {@link OsVersion} and its upgrade budget.
+ * The OS version target for a cloud/system, containing the {@link OsVersion} and its upgrade budget.
  *
  * @author mpolden
  */
-public class OsVersionTarget implements Comparable<OsVersionTarget> {
+public class OsVersionTarget implements VersionTarget, Comparable<OsVersionTarget> {
 
     // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one
     //          (and rewrite all nodes on startup), changes to the serialized format must be made
@@ -63,4 +65,14 @@ public class OsVersionTarget implements Comparable<OsVersionTarget> {
         return osVersion.compareTo(o.osVersion);
     }
 
+    @Override
+    public Version version() {
+        return osVersion.version();
+    }
+
+    @Override
+    public boolean downgrade() {
+        return false; // Not supported by this target type
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionTarget.java
new file mode 100644
index 00000000000..9b53d04c80f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionTarget.java
@@ -0,0 +1,19 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.yahoo.component.Version;
+
+/**
+ * Interface for a version target of some kind of upgrade.
+ *
+ * @author mpolden
+ */
+public interface VersionTarget {
+
+    /** The version of this target */
+    Version version();
+
+    /** Returns whether this target is potentially a downgrade */
+    boolean downgrade();
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index 660704d356b..792ec36bbb3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -130,6 +130,9 @@ public class VespaVersion implements Comparable<VespaVersion> {
     /** The confidence of a version. */
     public enum Confidence {
 
+        /** Rollout was aborted. The system infrastructure should stay on, or roll back to, its current version */
+        aborted,
+
         /** This version has been proven defective */
         broken,
         
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java
new file mode 100644
index 00000000000..fd5603b96b8
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersionTarget.java
@@ -0,0 +1,33 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.yahoo.component.Version;
+
+import java.util.Objects;
+
+/**
+ * The target Vespa version for a system.
+ *
+ * @author mpolden
+ */
+public class VespaVersionTarget implements VersionTarget {
+
+    private final Version version;
+    private final boolean downgrade;
+
+    public VespaVersionTarget(Version version, boolean downgrade) {
+        this.version = Objects.requireNonNull(version);
+        this.downgrade = downgrade;
+    }
+
+    @Override
+    public Version version() {
+        return version;
+    }
+
+    @Override
+    public boolean downgrade() {
+        return downgrade;
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 6697ddac808..99c97b3bdd6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -251,7 +251,7 @@ public final class ControllerTester {
         for (ZoneApi zone : zoneRegistry().zones().all().zones()) {
             for (SystemApplication application : systemApplications) {
                 if (!application.hasApplicationPackage()) {
-                    configServer().nodeRepository().upgrade(zone.getId(), application.nodeType(), version);
+                    configServer().nodeRepository().upgrade(zone.getId(), application.nodeType(), version, false);
                 }
                 configServer().setVersion(version, application.id(), zone.getId());
                 configServer().convergeServices(application.id(), zone.getId());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
index ef99183ecde..a2a1b4ba0a1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java
@@ -125,11 +125,17 @@ public class NodeRepositoryMock implements NodeRepository {
     }
 
     @Override
-    public void upgrade(ZoneId zone, NodeType type, Version version) {
+    public void upgrade(ZoneId zone, NodeType type, Version version, boolean allowDowngrade) {
         this.targetVersions.compute(zone, (ignored, targetVersions) -> {
             if (targetVersions == null) {
                 targetVersions = TargetVersions.EMPTY;
             }
+            Optional<Version> current = targetVersions.vespaVersion(type);
+            if (current.isPresent() && version.isBefore(current.get()) && !allowDowngrade) {
+                throw new IllegalArgumentException("Changing wanted version for " + type + " in " + zone + " from " +
+                                                   current.get() + " to " + version +
+                                                   ", but downgrade is not allowed");
+            }
             return targetVersions.withVespaVersion(type, version);
         });
         // Bump wanted version of each node. This is done by InfrastructureProvisioner in a real node repository.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index 71a3ce262ad..604a42f3d19 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -295,7 +295,7 @@ public class MetricsReporterTest {
         var tester = new ControllerTester();
         var reporter = createReporter(tester.controller());
         var zone = ZoneId.from("prod.eu-west-1");
-        tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.create().upgrade(ZoneApiMock.from(zone)));
+        tester.zoneRegistry().setUpgradePolicy(UpgradePolicy.builder().upgrade(ZoneApiMock.from(zone)).build());
         var systemUpgrader = new SystemUpgrader(tester.controller(), Duration.ofDays(1)
         );
         tester.configServer().bootstrap(List.of(zone), SystemApplication.configServer);
@@ -352,7 +352,7 @@ public class MetricsReporterTest {
         var reporter = createReporter(tester.controller());
         var zone = ZoneId.from("prod.eu-west-1");
         var cloud = CloudName.defaultName();
-        tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.create().upgrade(ZoneApiMock.from(zone)));
+        tester.zoneRegistry().setOsUpgradePolicy(cloud, UpgradePolicy.builder().upgrade(ZoneApiMock.from(zone)).build());
         var osUpgrader = new OsUpgrader(tester.controller(), Duration.ofDays(1), CloudName.defaultName());
         var statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1)
         );
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
index ec1a5455413..3c3f0053e91 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
@@ -42,12 +42,13 @@ public class OsUpgraderTest {
         ZoneApi zone3 = zone("prod.us-central-1", cloud1);
         ZoneApi zone4 = zone("prod.us-east-3", cloud1);
         ZoneApi zone5 = zone("prod.us-north-1", cloud2);
-        UpgradePolicy upgradePolicy = UpgradePolicy.create()
+        UpgradePolicy upgradePolicy = UpgradePolicy.builder()
                                                    .upgrade(zone0)
                                                    .upgrade(zone1)
                                                    .upgradeInParallel(zone2, zone3)
                                                    .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader
-                                                   .upgrade(zone4);
+                                                   .upgrade(zone4)
+                                                   .build();
         OsUpgrader osUpgrader = osUpgrader(upgradePolicy, cloud1, false);
 
         // Bootstrap system
@@ -125,11 +126,12 @@ public class OsUpgraderTest {
         ZoneApi zone2 = zone("prod.us-west-1", cloud);
         ZoneApi zone3 = zone("prod.us-central-1", cloud);
         ZoneApi zone4 = zone("prod.eu-west-1", cloud);
-        UpgradePolicy upgradePolicy = UpgradePolicy.create()
+        UpgradePolicy upgradePolicy = UpgradePolicy.builder()
                                                    .upgrade(zone0)
                                                    .upgrade(zone1)
                                                    .upgradeInParallel(zone2, zone3)
-                                                   .upgrade(zone4);
+                                                   .upgrade(zone4)
+                                                   .build();
         OsUpgrader osUpgrader = osUpgrader(upgradePolicy, cloud, true);
 
         // Bootstrap system
@@ -189,9 +191,10 @@ public class OsUpgraderTest {
         CloudName cloud = CloudName.from("cloud");
         ZoneApi zone1 = zone("dev.us-east-1", cloud);
         ZoneApi zone2 = zone("prod.us-west-1", cloud);
-        UpgradePolicy upgradePolicy = UpgradePolicy.create()
+        UpgradePolicy upgradePolicy = UpgradePolicy.builder()
                                                    .upgrade(zone1)
-                                                   .upgrade(zone2);
+                                                   .upgrade(zone2)
+                                                   .build();
         OsUpgrader osUpgrader = osUpgrader(upgradePolicy, cloud, false);
 
         // Bootstrap system
@@ -299,7 +302,7 @@ public class OsUpgraderTest {
     }
 
     private OsUpgrader osUpgrader(UpgradePolicy upgradePolicy, CloudName cloud, boolean reprovisionToUpgradeOs) {
-        var zones = upgradePolicy.asList().stream().flatMap(Collection::stream).collect(Collectors.toList());
+        var zones = upgradePolicy.steps().stream().flatMap(Collection::stream).collect(Collectors.toList());
         tester.zoneRegistry()
               .setZones(zones)
               .setOsUpgradePolicy(cloud, upgradePolicy);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
index f5ae6bafc65..f2d738bd2e1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
@@ -28,11 +28,11 @@ public class OsVersionStatusUpdaterTest {
         OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1)
         );
         // Add all zones to upgrade policy
-        UpgradePolicy upgradePolicy = UpgradePolicy.create();
+        UpgradePolicy.Builder upgradePolicy = UpgradePolicy.builder();
         for (ZoneApi zone : tester.zoneRegistry().zones().controllerUpgraded().zones()) {
             upgradePolicy = upgradePolicy.upgrade(zone);
         }
-        tester.zoneRegistry().setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy);
+        tester.zoneRegistry().setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy.build());
 
         // Initially empty
         assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index ff47b6cc231..c09d3ec3a92 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -21,6 +21,7 @@ import java.util.stream.Stream;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
 
 /**
  * @author mpolden
@@ -37,10 +38,11 @@ public class SystemUpgraderTest {
     @Test
     public void upgrade_system() {
         SystemUpgrader systemUpgrader = systemUpgrader(
-                UpgradePolicy.create()
+                UpgradePolicy.builder()
                              .upgrade(zone1)
                              .upgradeInParallel(zone2, zone3)
                              .upgrade(zone4)
+                             .build()
         );
 
         Version version1 = Version.fromString("6.5");
@@ -137,7 +139,7 @@ public class SystemUpgraderTest {
 
     @Test
     public void upgrade_controller_with_non_converging_application() {
-        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1));
+        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.builder().upgrade(zone1).build());
 
         // Bootstrap system
         tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer,
@@ -173,10 +175,11 @@ public class SystemUpgraderTest {
     @Test
     public void upgrade_system_containing_host_applications() {
         SystemUpgrader systemUpgrader = systemUpgrader(
-                UpgradePolicy.create()
+                UpgradePolicy.builder()
                              .upgrade(zone1)
                              .upgradeInParallel(zone2, zone3)
                              .upgrade(zone4)
+                             .build()
         );
 
         Version version1 = Version.fromString("6.5");
@@ -223,7 +226,7 @@ public class SystemUpgraderTest {
 
     @Test
     public void downgrading_controller_never_downgrades_system() {
-        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1));
+        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.builder().upgrade(zone1).build());
 
         Version version = Version.fromString("6.5");
         tester.upgradeSystem(version);
@@ -242,7 +245,7 @@ public class SystemUpgraderTest {
 
     @Test
     public void upgrade_halts_on_broken_version() {
-        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1).upgrade(zone2));
+        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.builder().upgrade(zone1).upgrade(zone2).build());
 
         // Initial system version
         Version version1 = Version.fromString("6.5");
@@ -267,9 +270,7 @@ public class SystemUpgraderTest {
         convergeServices(SystemApplication.proxy, zone1);
 
         // Confidence is reduced to broken and next zone is not scheduled for upgrade
-        new Upgrader(tester.controller(), Duration.ofDays(1))
-                .overrideConfidence(version2, VespaVersion.Confidence.broken);
-        tester.computeVersionStatus();
+        overrideConfidence(version2, VespaVersion.Confidence.broken);
         systemUpgrader.maintain();
         assertWantedVersion(List.of(SystemApplication.configServerHost, SystemApplication.proxyHost,
                                     SystemApplication.configServer, SystemApplication.proxy), version1, zone2);
@@ -282,7 +283,7 @@ public class SystemUpgraderTest {
         tester.configServer().bootstrap(List.of(zone1.getId()), applications);
         tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id());
         tester.zoneRegistry().exclusiveRoutingIn(zone1);
-        var systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1));
+        var systemUpgrader = systemUpgrader(UpgradePolicy.builder().upgrade(zone1).build());
 
         // System begins upgrade
         var version1 = Version.fromString("6.5");
@@ -299,6 +300,75 @@ public class SystemUpgraderTest {
         assertEquals(version1, tester.controller().readSystemVersion());
     }
 
+    @Test
+    public void downgrade_from_aborted_version() {
+        SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.builder().upgrade(zone1).upgrade(zone2).upgrade(zone3).build());
+
+        Version version1 = Version.fromString("6.5");
+        tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId()), SystemApplication.notController());
+        tester.upgradeSystem(version1);
+        systemUpgrader.maintain();
+        assertCurrentVersion(SystemApplication.notController(), version1, zone1, zone2, zone3);
+
+        // Controller upgrades
+        Version version2 = Version.fromString("6.6");
+        tester.upgradeController(version2);
+        assertControllerVersion(version2);
+
+        // 2/3 zones upgrade
+        for (var zone : List.of(zone1, zone2)) {
+            systemUpgrader.maintain();
+            completeUpgrade(List.of(SystemApplication.tenantHost,
+                                    SystemApplication.proxyHost,
+                                    SystemApplication.configServerHost),
+                            version2, zone);
+            completeUpgrade(SystemApplication.configServer, version2, zone);
+            systemUpgrader.maintain();
+            completeUpgrade(SystemApplication.proxy, version2, zone);
+            convergeServices(SystemApplication.proxy, zone);
+        }
+
+        // Upgrade is aborted
+        overrideConfidence(version2, VespaVersion.Confidence.aborted);
+
+        // Dependency graph is inverted and applications without dependencies downgrade first. Upgrade policy is
+        // also followed in inverted order
+        for (var zone : List.of(zone2, zone1)) {
+            systemUpgrader.maintain();
+            completeUpgrade(List.of(SystemApplication.tenantHost,
+                                    SystemApplication.configServerHost,
+                                    SystemApplication.proxy),
+                            version1, zone);
+            convergeServices(SystemApplication.proxy, zone);
+            List<SystemApplication> lastToDowngrade = List.of(SystemApplication.configServer,
+                                                              SystemApplication.proxyHost);
+            assertWantedVersion(lastToDowngrade, version2, zone);
+
+            // ... and then configserver and proxyhost
+            systemUpgrader.maintain();
+            completeUpgrade(lastToDowngrade, version1, zone);
+        }
+        assertSystemVersion(version1);
+
+        // Another version is released and system upgrades
+        Version version3 = Version.fromString("6.7");
+        tester.upgradeSystem(version3);
+        assertEquals(version3, tester.controller().readSystemVersion());
+
+        // Attempt to abort current system version is rejected
+        try {
+            overrideConfidence(version3, VespaVersion.Confidence.aborted);
+            fail("Expected exception");
+        } catch (IllegalArgumentException ignored) {}
+        systemUpgrader.maintain();
+        assertWantedVersion(SystemApplication.notController(), version3, zone1, zone2, zone3);
+    }
+
+    private void overrideConfidence(Version version, VespaVersion.Confidence confidence) {
+        new Upgrader(tester.controller(), Duration.ofDays(1)).overrideConfidence(version, confidence);
+        tester.computeVersionStatus();
+    }
+
     /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */
     private void completeUpgrade(SystemApplication application, Version version, ZoneApi first, ZoneApi... rest) {
         assertWantedVersion(application, version, first, rest);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
index 6b6ced68b0a..7d17e97e66b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
@@ -55,8 +55,8 @@ public class OsApiTest extends ControllerContainerTest {
         zoneRegistryMock().setSystemName(SystemName.cd)
                           .setZones(zone1, zone2, zone3)
                           .reprovisionToUpgradeOsIn(zone3)
-                          .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2))
-                          .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3));
+                          .setOsUpgradePolicy(cloud1, UpgradePolicy.builder().upgrade(zone1).upgrade(zone2).build())
+                          .setOsUpgradePolicy(cloud2, UpgradePolicy.builder().upgrade(zone3).build());
         osUpgraders = List.of(
                 new OsUpgrader(tester.controller(), Duration.ofDays(1),
                                cloud1),