Merge pull request #6638 from vespa-engine/mpolden/os-upgrade-per-cloud

Support per-cloud OS upgrades

19 files changed, 455 insertions, 202 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java
index c98f6da3f29..1b13d9a5760 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneId.java
@@ -21,9 +21,9 @@ public class ZoneId {
     private final CloudName cloud;
 
     private ZoneId(Environment environment, RegionName region, CloudName cloud) {
-        this.environment = Objects.requireNonNull(environment);
-        this.region = Objects.requireNonNull(region);
-        this.cloud = cloud;
+        this.environment = Objects.requireNonNull(environment, "environment must be non-null");
+        this.region = Objects.requireNonNull(region, "region must be non-null");
+        this.cloud = Objects.requireNonNull(cloud, "cloud must be non-null");
     }
 
     private ZoneId(Environment environment, RegionName region) {
@@ -80,13 +80,12 @@ public class ZoneId {
         if (o == null || getClass() != o.getClass()) return false;
         ZoneId zoneId = (ZoneId) o;
         return environment == zoneId.environment &&
-               Objects.equals(region, zoneId.region) &&
-               Objects.equals(cloud, zoneId.cloud);
+               Objects.equals(region, zoneId.region);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(environment, region, cloud);
+        return Objects.hash(environment, region);
     }
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
index 4fbad88df8d..419e532c531 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/zone/ZoneRegistry.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 
 import java.net.URI;
 import java.time.Duration;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 
@@ -58,6 +59,12 @@ public interface ZoneRegistry {
         return upgradePolicy();
     }
 
+    // TODO: Remove default implementation
+    /** Returns all OS upgrade policies */
+    default List<UpgradePolicy> osUpgradePolicies() {
+        return Collections.singletonList(upgradePolicy());
+    }
+
     /** Returns the OS upgrade policy to use for zones belonging to given cloud, in this registry */
     default UpgradePolicy osUpgradePolicy(CloudName cloud) {
         return osUpgradePolicy(); // TODO: Remove default implementation
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 95542141e47..3733022766e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -26,10 +26,12 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.Organizati
 import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingService;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.rotation.Rotation;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
@@ -42,6 +44,8 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.Set;
+import java.util.TreeSet;
 import java.util.function.Predicate;
 import java.util.function.Supplier;
 import java.util.logging.Logger;
@@ -217,14 +221,22 @@ public class Controller extends AbstractComponent {
 
     /** Returns the target OS version for infrastructure in this system. The controller will drive infrastructure OS
      * upgrades to this version */
-    public Optional<Version> osVersion() {
-        return curator.readOsTargetVersion();
+    public Optional<OsVersion> osVersion(CloudName cloud) {
+        return osVersions().stream().filter(osVersion -> osVersion.cloud().equals(cloud)).findFirst();
     }
 
-    /** Set the target OS version for infrastructure in this system. */
-    public void upgradeOs(Version version) {
-        try (Lock lock = curator.lockOsTargetVersion()) {
-            curator.writeOsTargetVersion(version);
+    /** Returns all target OS versions in this system */
+    public Set<OsVersion> osVersions() {
+        return curator.readOsVersions();
+    }
+
+    /** Set the target OS version for infrastructure on cloud in this system */
+    public void upgradeOsIn(CloudName cloud, Version version) {
+        try (Lock lock = curator.lockOsVersions()) {
+            Set<OsVersion> versions = new TreeSet<>(curator.readOsVersions());
+            versions.removeIf(osVersion -> osVersion.cloud().equals(cloud)); // Only allow a single target per cloud
+            versions.add(new OsVersion(version, cloud));
+            curator.writeOsVersions(versions);
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
index fedd38c56b8..cc4f236f3b2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.stream.Collectors;
 
 /**
  * This represents a system-level application in hosted Vespa. E.g. the zone-application.
@@ -59,7 +60,7 @@ public enum SystemApplication {
 
     /** Returns whether this system application has an application package */
     public boolean hasApplicationPackage() {
-        return nodeTypes.contains(NodeType.proxy);
+        return this == zone;
     }
 
     /** Returns whether config for this application has converged in given zone */
@@ -78,11 +79,7 @@ public enum SystemApplication {
 
     /** Returns the node types of this that should receive OS upgrades */
     public Set<NodeType> nodeTypesWithUpgradableOs() {
-        // TODO: Change this to include all node types that are Docker hosts
-        if (this != zone) {
-            return Collections.emptySet();
-        }
-        return Collections.singleton(NodeType.host);
+        return nodeTypes().stream().filter(NodeType::isDockerHost).collect(Collectors.toSet());
     }
 
     /** All known system applications */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 0214eb3cd6c..fd71020343e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -10,11 +10,15 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.time.Duration;
+import java.util.Collections;
+import java.util.List;
+import java.util.stream.Collectors;
 
 /**
  * Maintenance jobs of the controller.
@@ -40,7 +44,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer;
     private final DnsMaintainer dnsMaintainer;
     private final SystemUpgrader systemUpgrader;
-    private final OsUpgrader osUpgrader;
+    private final List<OsUpgrader> osUpgraders;
     private final OsVersionStatusUpdater osVersionStatusUpdater;
     private final JobRunner jobRunner;
 
@@ -65,7 +69,7 @@ public class ControllerMaintenance extends AbstractComponent {
         dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService);
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
         jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testerCloud));
-        osUpgrader = new OsUpgrader(controller, Duration.ofMinutes(1), jobControl);
+        osUpgraders = osUpgraders(controller, jobControl);
         osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl);
     }
 
@@ -89,9 +93,19 @@ public class ControllerMaintenance extends AbstractComponent {
         applicationOwnershipConfirmer.deconstruct();
         dnsMaintainer.deconstruct();
         systemUpgrader.deconstruct();
-        osUpgrader.deconstruct();
+        osUpgraders.forEach(Maintainer::deconstruct);
         osVersionStatusUpdater.deconstruct();
         jobRunner.deconstruct();
     }
 
+    /** Create one OS upgrader per cloud found in the zone registry of controller */
+    private static List<OsUpgrader> osUpgraders(Controller controller, JobControl jobControl) {
+        return controller.zoneRegistry().zones().controllerUpgraded().ids().stream()
+                         .map(ZoneId::cloud)
+                         .distinct()
+                         .sorted()
+                         .map(cloud -> new OsUpgrader(controller, Duration.ofMinutes(1), jobControl, cloud))
+                         .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
index 9c4deaa8370..a8e70aedacb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java
@@ -27,8 +27,9 @@ public abstract class InfrastructureUpgrader extends Maintainer {
 
     private final UpgradePolicy upgradePolicy;
 
-    public InfrastructureUpgrader(Controller controller, Duration interval, JobControl jobControl, UpgradePolicy upgradePolicy) {
-        super(controller, interval, jobControl);
+    public InfrastructureUpgrader(Controller controller, Duration interval, JobControl jobControl,
+                                  UpgradePolicy upgradePolicy, String name) {
+        super(controller, interval, jobControl, name);
         this.upgradePolicy = upgradePolicy;
     }
 
@@ -74,26 +75,26 @@ public abstract class InfrastructureUpgrader extends Maintainer {
         return applications.stream().allMatch(application -> convergedOn(target, application, zone));
     }
 
-    /** Upgrade components to target version. Implementation should be idempotent */
+    /** Upgrade component to target version. Implementation should be idempotent */
     protected abstract void upgrade(Version target, SystemApplication application, ZoneId zone);
 
     /** Returns whether application has converged to target version in zone */
     protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneId zone);
 
-    /** Returns target version for components upgraded by this */
+    /** Returns the target version for the component upgraded by this, if any */
     protected abstract Optional<Version> targetVersion();
 
-    /** Returns whether the upgrader should require given node to upgrade in application */
-    protected abstract boolean requireUpgradeOf(Node node, SystemApplication application);
+    /** Returns whether the upgrader should require given node to upgrade */
+    protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone);
 
     /** Find the minimum value of a version field in a zone */
-    protected Optional<Version> minVersion(ZoneId zone, SystemApplication application, Function<Node, Version> versionField) {
+    protected final Optional<Version> minVersion(ZoneId zone, SystemApplication application, Function<Node, Version> versionField) {
         try {
             return controller().configServer()
                                .nodeRepository()
                                .list(zone, application.id())
                                .stream()
-                               .filter((node) -> requireUpgradeOf(node, application))
+                               .filter((node) -> requireUpgradeOf(node, application, zone))
                                .map(versionField)
                                .min(Comparator.naturalOrder());
         } catch (Exception e) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
index f6ccbf6aa4e..d76f557fce1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Maintainer.java
@@ -1,12 +1,9 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
-import com.google.common.util.concurrent.UncheckedTimeoutException;
 import com.yahoo.component.AbstractComponent;
-import com.yahoo.component.ComponentId;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.time.Duration;
 import java.util.concurrent.ScheduledExecutorService;
@@ -29,11 +26,17 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
     private final Duration maintenanceInterval;
     private final JobControl jobControl;
     private final ScheduledExecutorService service;
+    private final String name;
 
     public Maintainer(Controller controller, Duration interval, JobControl jobControl) {
+        this(controller, interval, jobControl, null);
+    }
+
+    public Maintainer(Controller controller, Duration interval, JobControl jobControl, String name) {
         this.controller = controller;
         this.maintenanceInterval = interval;
         this.jobControl = jobControl;
+        this.name = name;
 
         service = new ScheduledThreadPoolExecutor(1);
         service.scheduleAtFixedRate(this, interval.toMillis(), interval.toMillis(), TimeUnit.MILLISECONDS);
@@ -69,7 +72,9 @@ public abstract class Maintainer extends AbstractComponent implements Runnable {
 
     public Duration maintenanceInterval() { return maintenanceInterval; }
     
-    public String name() { return this.getClass().getSimpleName(); }
+    public final String name() {
+        return name == null ? this.getClass().getSimpleName() : name;
+    }
     
     /** Returns the name of this */
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
index 2937c2a1fed..bf9fbeb26d3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java
@@ -6,12 +6,15 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 
 import java.time.Duration;
 import java.util.Optional;
 import java.util.Set;
+import java.util.logging.Logger;
 
 /**
  * Maintenance job that schedules upgrades of OS / kernel on nodes in the system.
@@ -20,14 +23,19 @@ import java.util.Set;
  */
 public class OsUpgrader extends InfrastructureUpgrader {
 
+    private static final Logger log = Logger.getLogger(OsUpgrader.class.getName());
+
     private static final Set<Node.State> upgradableNodeStates = ImmutableSet.of(
             Node.State.ready,
             Node.State.active,
             Node.State.reserved
     );
 
-    public OsUpgrader(Controller controller, Duration interval, JobControl jobControl) {
-        super(controller, interval, jobControl, controller.zoneRegistry().osUpgradePolicy());
+    private final CloudName cloud;
+
+    public OsUpgrader(Controller controller, Duration interval, JobControl jobControl, CloudName cloud) {
+        super(controller, interval, jobControl, controller.zoneRegistry().osUpgradePolicy(cloud), name(cloud));
+        this.cloud = cloud;
     }
 
     @Override
@@ -38,6 +46,8 @@ public class OsUpgrader extends InfrastructureUpgrader {
 
     @Override
     protected void upgrade(Version target, SystemApplication application, ZoneId zone) {
+        log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone));
+        // Node repository ensures the upgrade call is idempotent
         application.nodeTypesWithUpgradableOs().forEach(nodeType -> controller().configServer().nodeRepository()
                                                                                 .upgradeOs(zone, nodeType, target));
     }
@@ -48,26 +58,42 @@ public class OsUpgrader extends InfrastructureUpgrader {
     }
 
     @Override
-    protected boolean requireUpgradeOf(Node node, SystemApplication application) {
-        return eligibleForUpgrade(node, application);
-    }
-
-    private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) {
-        return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion);
+    protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) {
+        return cloud.equals(zone.cloud()) && eligibleForUpgrade(node, application);
     }
 
     @Override
     protected Optional<Version> targetVersion() {
-        // Only schedule upgrades if we have nodes in the system on a lower version
-        return controller().curator().readOsTargetVersion().filter(
-                target -> controller().osVersionStatus().versions().stream()
-                                      .anyMatch(osVersion -> osVersion.version().isBefore(target))
-        );
+        // Return target if we have nodes in this cloud on a lower version
+        return controller().osVersion(cloud)
+                           .filter(target -> controller().osVersionStatus().nodeVersionsIn(cloud).stream()
+                                                         .anyMatch(node -> node.version().isBefore(target.version())))
+                           .map(OsVersion::version);
+    }
+
+    private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) {
+        return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion);
     }
 
     /** Returns whether node in application should be upgraded by this */
     public static boolean eligibleForUpgrade(Node node, SystemApplication application) {
-        return upgradableNodeStates.contains(node.state()) && application.nodeTypesWithUpgradableOs().contains(node.type());
+        return upgradableNodeStates.contains(node.state()) &&
+               application.nodeTypesWithUpgradableOs().contains(node.type());
+    }
+
+    private static String name(CloudName cloud) {
+        return capitalize(cloud.value()) + OsUpgrader.class.getSimpleName(); // Prefix maintainer name with cloud name
+    }
+
+    private static String capitalize(String s) {
+        if (s.isEmpty()) {
+            return s;
+        }
+        char firstLetter = Character.toUpperCase(s.charAt(0));
+        if (s.length() > 1) {
+            return firstLetter + s.substring(1).toLowerCase();
+        }
+        return String.valueOf(firstLetter);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
index afeca83d0d8..5c9c39e645e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
@@ -26,7 +26,7 @@ public class SystemUpgrader extends InfrastructureUpgrader {
     private static final Set<Node.State> upgradableNodeStates = ImmutableSet.of(Node.State.active, Node.State.reserved);
 
     public SystemUpgrader(Controller controller, Duration interval, JobControl jobControl) {
-        super(controller, interval, jobControl, controller.zoneRegistry().upgradePolicy());
+        super(controller, interval, jobControl, controller.zoneRegistry().upgradePolicy(), null);
     }
 
     @Override
@@ -44,7 +44,7 @@ public class SystemUpgrader extends InfrastructureUpgrader {
     }
 
     @Override
-    protected boolean requireUpgradeOf(Node node, SystemApplication application) {
+    protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) {
         return eligibleForUpgrade(node);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 8bc0f5786c6..2ed69ad9be8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.deployment.Step;
 import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.controller.tenant.UserTenant;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
@@ -71,7 +72,8 @@ public class CuratorDb {
     private final TenantSerializer tenantSerializer = new TenantSerializer();
     private final ApplicationSerializer applicationSerializer = new ApplicationSerializer();
     private final RunSerializer runSerializer = new RunSerializer();
-    private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer();
+    private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer();
+    private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer);
 
     private final Curator curator;
 
@@ -151,7 +153,7 @@ public class CuratorDb {
         return lock(lockRoot.append("openStackServerPoolLock"), Duration.ofSeconds(1));
     }
 
-    public Lock lockOsTargetVersion() {
+    public Lock lockOsVersions() {
         return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout);
     }
 
@@ -247,12 +249,12 @@ public class CuratorDb {
 
     // Infrastructure upgrades
 
-    public void writeOsTargetVersion(Version version) {
-        curator.set(osTargetVersionPath(), asJson(versionSerializer.toSlime(version)));
+    public void writeOsVersions(Set<OsVersion> versions) {
+        curator.set(osTargetVersionPath(), asJson(osVersionSerializer.toSlime(versions)));
     }
 
-    public Optional<Version> readOsTargetVersion() {
-        return readSlime(osTargetVersionPath()).map(versionSerializer::fromSlime);
+    public Set<OsVersion> readOsVersions() {
+        return readSlime(osTargetVersionPath()).map(osVersionSerializer::fromSlime).orElseGet(Collections::emptySet);
     }
 
     public void writeOsVersionStatus(OsVersionStatus status) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java
new file mode 100644
index 00000000000..aee19644b82
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java
@@ -0,0 +1,62 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+
+import java.util.Collections;
+import java.util.Set;
+import java.util.TreeSet;
+
+/**
+ * Serializer for an OS version.
+ *
+ * @author mpolden
+ */
+public class OsVersionSerializer {
+
+    private static final String versionsField = "versions";
+    private static final String versionField = "version";
+    private static final String cloudField = "cloud";
+
+    public Slime toSlime(Set<OsVersion> osVersions) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Cursor array = root.setArray(versionsField);
+        osVersions.forEach(osVersion -> toSlime(osVersion, array.addObject()));
+        return slime;
+    }
+
+    public void toSlime(OsVersion osVersion, Cursor object) {
+        object.setString(versionField, osVersion.version().toFullString());
+        object.setString(cloudField, osVersion.cloud().value());
+    }
+
+    public Set<OsVersion> fromSlime(Slime slime) {
+        Inspector array = slime.get().field(versionsField);
+        Set<OsVersion> osVersions = new TreeSet<>();
+        array.traverse((ArrayTraverser) (i, inspector) -> osVersions.add(fromSlime(inspector)));
+        return Collections.unmodifiableSet(osVersions);
+    }
+
+    public OsVersion fromSlime(Inspector object) {
+        return new OsVersion(
+                Version.fromString(object.field(versionField).asString()),
+                cloudFrom(object.field(cloudField))
+        );
+    }
+
+    // TODO: Simplify and inline after 2018-09-01
+    private static CloudName cloudFrom(Inspector field) {
+        if (!field.valid()) {
+            return CloudName.defaultName();
+        }
+        return CloudName.from(field.asString());
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java
index 19beca661be..b0557863426 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java
@@ -15,6 +15,9 @@ import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.TreeMap;
 
 /**
  * Serializer for OS version status.
@@ -30,11 +33,21 @@ public class OsVersionStatusSerializer {
     private static final String regionField = "region";
     private static final String environmentField = "environment";
 
+    private final OsVersionSerializer osVersionSerializer;
+
+    public OsVersionStatusSerializer(OsVersionSerializer osVersionSerializer) {
+        this.osVersionSerializer = Objects.requireNonNull(osVersionSerializer, "osVersionSerializer must be non-null");
+    }
+
     public Slime toSlime(OsVersionStatus status) {
         Slime slime = new Slime();
         Cursor root = slime.setObject();
         Cursor versions = root.setArray(versionsField);
-        status.versions().forEach(version -> osVersionToSlime(version, versions.addObject()));
+        status.versions().forEach((version, nodes) -> {
+            Cursor object = versions.addObject();
+            osVersionSerializer.toSlime(version, object);
+            nodesToSlime(nodes, object.setArray(nodesField));
+        });
         return slime;
     }
 
@@ -42,43 +55,35 @@ public class OsVersionStatusSerializer {
         return new OsVersionStatus(osVersionsFromSlime(slime.get().field(versionsField)));
     }
 
-    private void osVersionToSlime(OsVersion version, Cursor object) {
-        object.setString(versionField, version.version().toFullString());
-        nodesToSlime(version.nodes(), object.setArray(nodesField));
-    }
-
-    private void nodesToSlime(List<OsVersion.Node> nodes, Cursor array) {
+    private void nodesToSlime(List<OsVersionStatus.Node> nodes, Cursor array) {
         nodes.forEach(node -> nodeToSlime(node, array.addObject()));
     }
 
-    private void nodeToSlime(OsVersion.Node node, Cursor object) {
+    private void nodeToSlime(OsVersionStatus.Node node, Cursor object) {
         object.setString(hostnameField, node.hostname().value());
         object.setString(versionField, node.version().toFullString());
         object.setString(regionField, node.region().value());
         object.setString(environmentField, node.environment().value());
     }
 
-    private List<OsVersion> osVersionsFromSlime(Inspector array) {
-        List<OsVersion> versions = new ArrayList<>();
-        array.traverse((ArrayTraverser) (i, object) -> versions.add(osVersionFromSlime(object)));
-        return Collections.unmodifiableList(versions);
-    }
-
-    private OsVersion osVersionFromSlime(Inspector object) {
-        return new OsVersion(
-                Version.fromString(object.field(versionField).asString()),
-                nodesFromSlime(object.field(nodesField))
-        );
+    private Map<OsVersion, List<OsVersionStatus.Node>> osVersionsFromSlime(Inspector array) {
+        Map<OsVersion, List<OsVersionStatus.Node>> versions = new TreeMap<>();
+        array.traverse((ArrayTraverser) (i, object) -> {
+            OsVersion osVersion = osVersionSerializer.fromSlime(object);
+            List<OsVersionStatus.Node> nodes = nodesFromSlime(object.field(nodesField));
+            versions.put(osVersion, nodes);
+        });
+        return Collections.unmodifiableMap(versions);
     }
 
-    private List<OsVersion.Node> nodesFromSlime(Inspector array) {
-        List<OsVersion.Node> nodes = new ArrayList<>();
+    private List<OsVersionStatus.Node> nodesFromSlime(Inspector array) {
+        List<OsVersionStatus.Node> nodes = new ArrayList<>();
         array.traverse((ArrayTraverser) (i, object) -> nodes.add(nodeFromSlime(object)));
         return Collections.unmodifiableList(nodes);
     }
 
-    private OsVersion.Node nodeFromSlime(Inspector object) {
-        return new OsVersion.Node(
+    private OsVersionStatus.Node nodeFromSlime(Inspector object) {
+        return new OsVersionStatus.Node(
                 HostName.from(object.field(hostnameField).asString()),
                 Version.fromString(object.field(versionField).asString()),
                 Environment.from(object.field(environmentField).asString()),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java
index e132236d07c..89009270f10 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersion.java
@@ -1,28 +1,25 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableList;
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import org.jetbrains.annotations.NotNull;
 
-import java.util.List;
 import java.util.Objects;
 
 /**
- * Information about a given OS version in the system.
+ * An OS version for a cloud in this system.
  *
  * @author mpolden
  */
-public class OsVersion {
+public class OsVersion implements Comparable<OsVersion> {
 
     private final Version version;
-    private final List<Node> nodes;
+    private final CloudName cloud;
 
-    public OsVersion(Version version, List<Node> nodes) {
-        this.version = version;
-        this.nodes = ImmutableList.copyOf(nodes);
+    public OsVersion(Version version, CloudName cloud) {
+        this.version = Objects.requireNonNull(version, "version must be non-null");
+        this.cloud = Objects.requireNonNull(cloud, "cloud must be non-null");
     }
 
     /** The version number of this */
@@ -30,56 +27,37 @@ public class OsVersion {
         return version;
     }
 
-    /** Nodes on this version */
-    public List<Node> nodes() {
-        return nodes;
+    /** The cloud where this OS version is used */
+    public CloudName cloud() {
+        return cloud;
     }
 
-    public static class Node {
-
-        private final HostName hostname;
-        private final Version version;
-        private final Environment environment;
-        private final RegionName region;
-
-        public Node(HostName hostname, Version version, Environment environment, RegionName region) {
-            this.hostname = hostname;
-            this.version = version;
-            this.environment = environment;
-            this.region = region;
-        }
-
-        public HostName hostname() {
-            return hostname;
-        }
-
-        public Version version() {
-            return version;
-        }
-
-        public Environment environment() {
-            return environment;
-        }
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        OsVersion osVersion = (OsVersion) o;
+        return Objects.equals(version, osVersion.version) &&
+               Objects.equals(cloud, osVersion.cloud);
+    }
 
-        public RegionName region() {
-            return region;
-        }
+    @Override
+    public int hashCode() {
+        return Objects.hash(version, cloud);
+    }
 
-        @Override
-        public boolean equals(Object o) {
-            if (this == o) return true;
-            if (o == null || getClass() != o.getClass()) return false;
-            Node node = (Node) o;
-            return Objects.equals(hostname, node.hostname) &&
-                   Objects.equals(version, node.version) &&
-                   environment == node.environment &&
-                   Objects.equals(region, node.region);
-        }
+    @Override
+    public String toString() {
+        return "version " + version + " for " + cloud + " cloud";
+    }
 
-        @Override
-        public int hashCode() {
-            return Objects.hash(hostname, version, environment, region);
+    @Override
+    public int compareTo(@NotNull OsVersion o) {
+        int cloudCmp = cloud.compareTo(o.cloud());
+        if (cloudCmp == 0) { // Same cloud, sort by version
+            return version.compareTo(o.version());
         }
+        return cloudCmp;
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
index 518394b46fc..871a7872a9c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
@@ -1,21 +1,28 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RegionName;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.collectingAndThen;
+
 /**
  * Information about OS versions in this system.
  *
@@ -23,36 +30,44 @@ import java.util.stream.Collectors;
  */
 public class OsVersionStatus {
 
-    public static final OsVersionStatus empty = new OsVersionStatus(Collections.emptyList());
+    public static final OsVersionStatus empty = new OsVersionStatus(Collections.emptyMap());
 
-    private final List<OsVersion> versions;
+    private final Map<OsVersion, List<Node>> versions;
 
-    public OsVersionStatus(List<OsVersion> versions) {
-        this.versions = ImmutableList.copyOf(versions);
+    /** Public for serialization purpose only. Use {@link OsVersionStatus#compute(Controller)} for an up-to-date status */
+    public OsVersionStatus(Map<OsVersion, List<Node>> versions) {
+        this.versions = ImmutableMap.copyOf(Objects.requireNonNull(versions, "versions must be non-null"));
     }
 
-    /** All known OS versions */
-    public List<OsVersion> versions() {
+    /** All known OS versions and their nodes */
+    public Map<OsVersion, List<Node>> versions() {
         return versions;
     }
 
-    /**
-     * Compute the current OS version status in this status. This is expensive as all config servers in the system
-     * must be queried.
-     */
+    /** Returns node versions that exist in given cloud */
+    public List<Node> nodeVersionsIn(CloudName cloud) {
+        return versions.entrySet().stream()
+                       .filter(entry -> entry.getKey().cloud().equals(cloud))
+                       .flatMap(entry -> entry.getValue().stream())
+                       .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
+    /** Compute the current OS versions in this system. This is expensive and should be called infrequently */
     public static OsVersionStatus compute(Controller controller) {
-        Map<Version, List<OsVersion.Node>> versions = new HashMap<>();
-        // Always include target version, if set
-        controller.osVersion().ifPresent(version -> versions.put(version, new ArrayList<>()));
+        Map<OsVersion, List<Node>> versions = new HashMap<>();
+
+        // Always include all target versions
+        controller.osVersions().forEach(osVersion -> versions.put(osVersion, new ArrayList<>()));
+
         for (SystemApplication application : SystemApplication.all()) {
             if (application.nodeTypesWithUpgradableOs().isEmpty()) {
-                continue; // Avoid querying applications that do not have nodes with upgradable OS
+                continue; // Avoid querying applications that do not contain nodes with upgradable OS
             }
-            for (ZoneId zone : controller.zoneRegistry().zones().controllerUpgraded().ids()) {
+            for (ZoneId zone : zonesToUpgrade(controller)) {
                 controller.configServer().nodeRepository().list(zone, application.id()).stream()
                           .filter(node -> OsUpgrader.eligibleForUpgrade(node, application))
-                          .map(node -> new OsVersion.Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region()))
-                          .forEach(node -> versions.compute(node.version(), (ignored, nodes) -> {
+                          .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region()))
+                          .forEach(node -> versions.compute(new OsVersion(node.version(), zone.cloud()), (ignored, nodes) -> {
                               if (nodes == null) {
                                   nodes = new ArrayList<>();
                               }
@@ -61,11 +76,63 @@ public class OsVersionStatus {
                           }));
             }
         }
-        return new OsVersionStatus(versions.entrySet()
-                                           .stream()
-                                           .map(kv -> new OsVersion(kv.getKey(), kv.getValue()))
-                                           .sorted(Comparator.comparing(OsVersion::version))
-                                           .collect(Collectors.toList()));
+
+        return new OsVersionStatus(versions);
+    }
+
+    private static List<ZoneId> zonesToUpgrade(Controller controller) {
+        return controller.zoneRegistry().osUpgradePolicies().stream()
+                         .flatMap(upgradePolicy -> upgradePolicy.asList().stream())
+                         .flatMap(Collection::stream)
+                         .collect(Collectors.toList());
+    }
+
+    /** A node in this system and its current OS version */
+    public static class Node {
+
+        private final HostName hostname;
+        private final Version version;
+        private final Environment environment;
+        private final RegionName region;
+
+        public Node(HostName hostname, Version version, Environment environment, RegionName region) {
+            this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null");
+            this.version = Objects.requireNonNull(version, "version must be non-null");
+            this.environment = Objects.requireNonNull(environment, "environment must be non-null");
+            this.region = Objects.requireNonNull(region, "region must be non-null");
+        }
+
+        public HostName hostname() {
+            return hostname;
+        }
+
+        public Version version() {
+            return version;
+        }
+
+        public Environment environment() {
+            return environment;
+        }
+
+        public RegionName region() {
+            return region;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Node node = (Node) o;
+            return Objects.equals(hostname, node.hostname) &&
+                   Objects.equals(version, node.version) &&
+                   environment == node.environment &&
+                   Objects.equals(region, node.region);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(hostname, version, environment, region);
+        }
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
index a243744a337..44823ab5777 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
@@ -5,17 +5,19 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.NodeType;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock;
-import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import org.junit.Before;
 import org.junit.Test;
 
 import java.time.Duration;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.function.Function;
@@ -34,6 +36,7 @@ public class OsUpgraderTest {
     private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1");
     private static final ZoneId zone3 = ZoneId.from("prod", "us-central-1");
     private static final ZoneId zone4 = ZoneId.from("prod", "us-east-3");
+    private static final ZoneId zone5 = ZoneId.from("prod", "us-north-1", "other");
 
     private DeploymentTester tester;
     private OsVersionStatusUpdater statusUpdater;
@@ -51,18 +54,19 @@ public class OsUpgraderTest {
                 UpgradePolicy.create()
                              .upgrade(zone1)
                              .upgradeInParallel(zone2, zone3)
-                             .upgrade(zone4)
+                             .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader
+                             .upgrade(zone4),
+                SystemName.cd
         );
 
         // Bootstrap system
-        tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4),
+        tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4, zone5),
                                         singletonList(SystemApplication.zone),
                                         Optional.of(NodeType.host));
 
         // Add system applications that exist in a real system, but are currently not upgraded
-        tester.configServer().addNodes(Arrays.asList(zone1, zone2, zone3, zone4),
-                                       Arrays.asList(SystemApplication.configServer,
-                                                     SystemApplication.configServerHost),
+        tester.configServer().addNodes(Arrays.asList(zone1, zone2, zone3, zone4, zone5),
+                                       Collections.singletonList(SystemApplication.configServer),
                                        Optional.empty());
 
         // Fail a few nodes. Failed nodes should not affect versions
@@ -71,7 +75,10 @@ public class OsUpgraderTest {
 
         // New OS version released
         Version version1 = Version.fromString("7.1");
-        tester.controller().upgradeOs(version1);
+        CloudName cloud = CloudName.defaultName();
+        tester.controller().upgradeOsIn(cloud, Version.fromString("7.0"));
+        tester.controller().upgradeOsIn(cloud, version1);
+        assertEquals(1, tester.controller().osVersions().size()); // Only allows one version per cloud
         statusUpdater.maintain();
 
         // zone 1: begins upgrading
@@ -85,7 +92,7 @@ public class OsUpgraderTest {
         completeUpgrade(version1, SystemApplication.zone, zone1);
         statusUpdater.maintain();
         assertEquals(2, nodesOn(version1).size());
-        assertEquals(8, nodesOn(Version.emptyVersion).size());
+        assertEquals(11, nodesOn(Version.emptyVersion).size());
 
         // zone 2 and 3: begins upgrading
         osUpgrader.maintain();
@@ -108,14 +115,41 @@ public class OsUpgraderTest {
         osUpgrader.maintain();
         assertWanted(version1, SystemApplication.zone, zone1, zone2, zone3, zone4);
         statusUpdater.maintain();
-        assertTrue("All nodes on target version", tester.controller().osVersionStatus().versions().stream()
-                                                        .allMatch(osVersion -> osVersion.version().equals(version1)));
+        assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodeVersionsIn(cloud).stream()
+                                                        .allMatch(node -> node.version().equals(version1)));
     }
 
-    private List<OsVersion.Node> nodesOn(Version version) {
-        return tester.controller().osVersionStatus().versions().stream()
-                     .filter(osVersion -> osVersion.version().equals(version))
-                     .flatMap(osVersion -> osVersion.nodes().stream())
+    // TODO: Remove once enabled in all systems
+    @Test
+    public void os_upgrade_in_main_does_nothing() {
+        OsUpgrader osUpgrader = osUpgrader(
+                UpgradePolicy.create()
+                             .upgrade(zone1)
+                             .upgradeInParallel(zone2, zone3)
+                             .upgrade(zone4),
+                SystemName.main
+        );
+
+        // Bootstrap system
+        tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4, zone5),
+                                        singletonList(SystemApplication.zone),
+                                        Optional.of(NodeType.host));
+
+        // New OS is released
+        CloudName cloud = CloudName.defaultName();
+        Version version1 = Version.fromString("7.1");
+        tester.controller().upgradeOsIn(cloud, version1);
+        statusUpdater.maintain();
+
+        // Nothing happens as main is explicitly disabled
+        osUpgrader.maintain();
+        assertWanted(Version.emptyVersion, SystemApplication.zone, zone1);
+    }
+
+    private List<OsVersionStatus.Node> nodesOn(Version version) {
+        return tester.controller().osVersionStatus().versions().entrySet().stream()
+                     .filter(entry -> entry.getKey().version().equals(version))
+                     .flatMap(entry -> entry.getValue().stream())
                      .collect(Collectors.toList());
     }
 
@@ -172,11 +206,13 @@ public class OsUpgraderTest {
         return tester.controllerTester().configServer().nodeRepository();
     }
 
-    private OsUpgrader osUpgrader(UpgradePolicy upgradePolicy) {
-        tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd);
-        tester.controllerTester().zoneRegistry().setUpgradePolicy(upgradePolicy);
+    private OsUpgrader osUpgrader(UpgradePolicy upgradePolicy, SystemName system) {
+        tester.controllerTester().zoneRegistry()
+              .setZones(zone1, zone2, zone3, zone4, zone5)
+              .setSystemName(system)
+              .setUpgradePolicy(upgradePolicy);
         return new OsUpgrader(tester.controller(), Duration.ofDays(1),
-                              new JobControl(tester.controllerTester().curator()));
+                              new JobControl(tester.controllerTester().curator()), CloudName.defaultName());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
index 48e1308a892..98ed64ba879 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
@@ -3,6 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
@@ -10,10 +13,12 @@ import org.junit.Test;
 
 import java.time.Duration;
 import java.util.List;
+import java.util.Map;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
 
 /**
  * @author mpolden
@@ -25,17 +30,26 @@ public class OsVersionStatusUpdaterTest {
         ControllerTester tester = new ControllerTester();
         OsVersionStatusUpdater statusUpdater = new OsVersionStatusUpdater(tester.controller(), Duration.ofDays(1),
                                                                           new JobControl(new MockCuratorDb()));
+        // Add all zones to upgrade policy
+        UpgradePolicy upgradePolicy = UpgradePolicy.create();
+        for (ZoneId zone : tester.zoneRegistry().zones().controllerUpgraded().ids()) {
+            upgradePolicy = upgradePolicy.upgrade(zone);
+        }
+        tester.zoneRegistry().setUpgradePolicy(upgradePolicy);
 
         // Initially empty
         assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus());
 
         // Setting a new target adds it to current status
         Version version1 = Version.fromString("7.1");
-        tester.controller().upgradeOs(version1);
+        CloudName cloud = CloudName.defaultName();
+        tester.controller().upgradeOsIn(cloud, version1);
         statusUpdater.maintain();
-        List<OsVersion> osVersions = tester.controller().osVersionStatus().versions();
-        assertFalse(osVersions.isEmpty());
-        assertEquals(version1, osVersions.get(0).version());
+
+        Map<OsVersion, List<OsVersionStatus.Node>> osVersions = tester.controller().osVersionStatus().versions();
+        assertEquals(2, osVersions.size());
+        assertFalse("All nodes on unknown version", osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty());
+        assertTrue("No nodes on current target", osVersions.get(new OsVersion(version1, cloud)).isEmpty());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java
new file mode 100644
index 00000000000..bd510cc9aef
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializerTest.java
@@ -0,0 +1,30 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.google.common.collect.ImmutableSet;
+import com.yahoo.component.Version;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import org.junit.Test;
+
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class OsVersionSerializerTest {
+
+    @Test
+    public void test_serialization() {
+        OsVersionSerializer serializer = new OsVersionSerializer();
+        Set<OsVersion> osVersions = ImmutableSet.of(
+                new OsVersion(Version.fromString("7.1"), CloudName.defaultName()),
+                new OsVersion(Version.fromString("7.1"), CloudName.from("foo"))
+        );
+        Set<OsVersion> serialized = serializer.fromSlime(serializer.toSlime(osVersions));
+        assertEquals(osVersions, serialized);
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java
index e0d3fe3ba29..9840bec0f9a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializerTest.java
@@ -5,12 +5,15 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.RegionName;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
 import com.yahoo.vespa.hosted.controller.versions.OsVersion;
 import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
 import org.junit.Test;
 
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 
 import static org.junit.Assert.assertEquals;
 
@@ -23,27 +26,22 @@ public class OsVersionStatusSerializerTest {
     public void test_serialization() {
         Version version1 = Version.fromString("7.1");
         Version version2 = Version.fromString("7.2");
-        List<OsVersion> osVersions = Arrays.asList(
-                new OsVersion(version1, Arrays.asList(
-                        new OsVersion.Node(HostName.from("node1"), version1, Environment.prod, RegionName.from("us-west")),
-                        new OsVersion.Node(HostName.from("node2"), version1, Environment.prod, RegionName.from("us-east"))
-                )),
-                new OsVersion(version2, Arrays.asList(
-                        new OsVersion.Node(HostName.from("node3"), version2, Environment.prod, RegionName.from("us-west")),
-                        new OsVersion.Node(HostName.from("node4"), version2, Environment.prod, RegionName.from("us-east"))
-                ))
-        );
-
-        OsVersionStatusSerializer serializer = new OsVersionStatusSerializer();
-        OsVersionStatus status = new OsVersionStatus(osVersions);
-        OsVersionStatus serialized = serializer.fromSlime(serializer.toSlime(status));
+        Map<OsVersion, List<OsVersionStatus.Node>> versions = new TreeMap<>();
+
+        versions.put(new OsVersion(version1, CloudName.defaultName()), Arrays.asList(
+                new OsVersionStatus.Node(HostName.from("node1"), version1, Environment.prod, RegionName.from("us-west")),
+                new OsVersionStatus.Node(HostName.from("node2"), version1, Environment.prod, RegionName.from("us-east"))
+        ));
+        versions.put(new OsVersion(version2, CloudName.defaultName()), Arrays.asList(
+                new OsVersionStatus.Node(HostName.from("node3"), version2, Environment.prod, RegionName.from("us-west")),
+                new OsVersionStatus.Node(HostName.from("node4"), version2, Environment.prod, RegionName.from("us-east"))
 
-        for (int i = 0; i < status.versions().size(); i++) {
-            OsVersion a = status.versions().get(i);
-            OsVersion b = serialized.versions().get(i);
-            assertEquals(a.version(), b.version());
-            assertEquals(a.nodes(), b.nodes());
-        }
+        ));
+
+        OsVersionStatusSerializer serializer = new OsVersionStatusSerializer(new OsVersionSerializer());
+        OsVersionStatus status = new OsVersionStatus(versions);
+        OsVersionStatus serialized = serializer.fromSlime(serializer.toSlime(status));
+        assertEquals(status.versions(), serialized.versions());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 5449e6387a9..2b847010482 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -10,6 +10,9 @@
       "name": "ClusterUtilizationMaintainer"
     },
     {
+      "name": "DefaultOsUpgrader"
+    },
+    {
       "name": "DeploymentExpirer"
     },
     {
@@ -28,9 +31,6 @@
       "name": "MetricsReporter"
     },
     {
-      "name": "OsUpgrader"
-    },
-    {
       "name": "OsVersionStatusUpdater"
     },
     {