Define and upgrade system applications

18 files changed, 432 insertions, 137 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 1f4c1412515..8e121e14dde 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -35,18 +35,18 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerato
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun;
-import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.tenant.Tenant;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
-import com.yahoo.vespa.hosted.controller.maintenance.DeploymentExpirer;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.rotation.Rotation;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
 import com.yahoo.vespa.hosted.controller.rotation.RotationRepository;
+import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
+import com.yahoo.vespa.hosted.controller.tenant.Tenant;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 import com.yahoo.yolean.Exceptions;
 
@@ -330,18 +330,36 @@ public class ApplicationController {
 
             // Carry out deployment
             options = withVersion(platformVersion, options);
-
-            DeploymentId deploymentId = new DeploymentId(applicationId, zone);
-            ConfigServer.PreparedApplication preparedApplication =
-                    configServer.deploy(deploymentId, options, cnames, rotationNames, applicationPackage.zippedContent());
-            // TODO: Set new deployment after convergence, rather than after deployment call, succeeds.
+            ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, cnames);
             application = application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant());
-
             store(application);
+            return result;
+        }
+    }
 
-            return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(),
-                                      applicationPackage.zippedContent().length);
+    /** Deploy a system application to given zone */
+    public void deploy(SystemApplication application, ZoneId zone, Version version) {
+        if (!application.hasApplicationPackage()) {
+            // Deploy by calling node repository directly
+            configServer().nodeRepository().upgrade(zone, application.nodeType(), version);
+            return;
         }
+        ApplicationPackage applicationPackage = new ApplicationPackage(
+                artifactRepository.getSystemApplicationPackage(application.id(), zone, version)
+        );
+        DeployOptions options = withVersion(version, DeployOptions.none());
+        deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet());
+    }
+
+    private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage,
+                                  ZoneId zone, DeployOptions deployOptions,
+                                  Set<String> rotationNames, Set<String> cnames) {
+        DeploymentId deploymentId = new DeploymentId(application, zone);
+        ConfigServer.PreparedApplication preparedApplication =
+                configServer.deploy(deploymentId, deployOptions, cnames, rotationNames,
+                                    applicationPackage.zippedContent());
+        return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(),
+                                  applicationPackage.zippedContent().length);
     }
 
     /** Makes sure the application has a global rotation, if eligible. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
new file mode 100644
index 00000000000..d3daa68741e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
@@ -0,0 +1,52 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.NodeType;
+
+import java.util.Arrays;
+import java.util.List;
+
+/**
+ * This represents a system-level application in hosted Vespa. E.g. the zone-application.
+ *
+ * @author mpolden
+ */
+public enum  SystemApplication {
+
+    configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config),
+    zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy);
+
+    private final ApplicationId id;
+    private final NodeType nodeType;
+
+    SystemApplication(ApplicationId id, NodeType nodeType) {
+        this.id = id;
+        this.nodeType = nodeType;
+    }
+
+    public ApplicationId id() {
+        return id;
+    }
+
+    /** The type of nodes that will be allocated to this */
+    public NodeType nodeType() {
+        return nodeType;
+    }
+
+    /** Returns whether this system application has its own application package */
+    public boolean hasApplicationPackage() {
+        return nodeType == NodeType.proxy;
+    }
+
+    /** All known system applications */
+    public static List<SystemApplication> all() {
+        return Arrays.asList(values());
+    }
+
+    @Override
+    public String toString() {
+        return String.format("system application %s of type %s", id, nodeType);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
index 3a729bb9657..a70bea9ac2b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java
@@ -2,18 +2,23 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
-import java.net.URI;
 import java.time.Duration;
+import java.util.Comparator;
 import java.util.List;
 import java.util.Optional;
+import java.util.function.Function;
+import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
- * Maintenance job which schedules upgrades of the system.
+ * Maintenance job which upgrades system applications.
  *
  * @author mpolden
  */
@@ -32,34 +37,50 @@ public class SystemUpgrader extends Maintainer {
             return;
         }
         for (List<ZoneId> zones : controller().zoneRegistry().upgradePolicy().asList()) {
-            if (!completeUpgrade(zones, target.get())) {
+            // The order here is important. Config servers should always upgrade first
+            if (!deploy(zones, SystemApplication.configServer, target.get())) {
+                break;
+            }
+            if (!deploy(zones, SystemApplication.zone, target.get())) {
                 break;
             }
         }
     }
 
-    /** Returns true if upgrade of given zones is complete */
-    private boolean completeUpgrade(List<ZoneId> zones, Version version) {
+    /** Deploy application on given version. Returns true when all allocated nodes are on requested version */
+    private boolean deploy(List<ZoneId> zones, SystemApplication application, Version version) {
         boolean completed = true;
         for (ZoneId zone : zones) {
-            startUpgrade(zone, version);
-            completed = completed && !isUpgrading(zone);
+            if (!wantedVersion(zone, application.id()).equals(version)) {
+                log.info(String.format("Deploying %s version %s in %s", application.id(), version, zone));
+                controller().applications().deploy(application, zone, version);
+            }
+            completed = completed && currentVersion(zone, application.id()).equals(version);
         }
         return completed;
     }
 
-    /** Returns true if any config servers in given zone are upgrading */
-    private boolean isUpgrading(ZoneId zone) {
-        return configServerUris(zone).stream().anyMatch(uri -> controller().configServer().version(uri).upgrading());
+    private Version wantedVersion(ZoneId zone, ApplicationId application) {
+        return minVersion(zone, application, Node::wantedVersion);
+    }
+
+    private Version currentVersion(ZoneId zone, ApplicationId application) {
+        return minVersion(zone, application, Node::currentVersion);
     }
 
-    /** Schedule upgrade of config servers in given zone, if necessary  */
-    private void startUpgrade(ZoneId zone, Version version) {
-        configServerUris(zone).stream()
-                              .filter(uri -> !controller().configServer().version(uri).wanted().equals(version))
-                              .peek(uri -> log.info(String.format("Upgrading config server %s in %s", uri.getHost(),
-                                                                  zone)))
-                              .forEach(uri -> controller().configServer().upgrade(uri, version));
+    private Version minVersion(ZoneId zone, ApplicationId application, Function<Node, Version> versionField) {
+        try {
+            return controller().configServer()
+                               .nodeRepository()
+                               .list(zone, application)
+                               .stream()
+                               .map(versionField)
+                               .min(Comparator.naturalOrder())
+                               .orElse(Version.emptyVersion);
+        } catch (Exception e) {
+            log.log(Level.WARNING, String.format("Failed to get version for %s in %s", application, zone), e);
+            return Version.emptyVersion;
+        }
     }
 
     /** Returns target version for the system */
@@ -69,8 +90,4 @@ public class SystemUpgrader extends Maintainer {
                            .map(VespaVersion::versionNumber);
     }
 
-    private List<URI> configServerUris(ZoneId zone) {
-        return controller().zoneRegistry().getConfigServerUris(zone);
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
index e0c443f2110..11f0bfcfa0f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
@@ -68,7 +68,7 @@ public class VersionStatusSerializer {
         object.setBool(isSystemVersionField, version.isSystemVersion());
         deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField));
         object.setString(confidenceField, version.confidence().name());
-        configServersToSlime(version.configServerHostnames(), object.setArray(configServersField));
+        configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField));
     }
 
     private void configServersToSlime(Set<HostName> configServerHostnames, Cursor array) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index ee0b9e678c6..2f2e72120ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -90,7 +90,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
             versionObject.setBool("systemVersion", version.isSystemVersion());
 
             Cursor configServerArray = versionObject.setArray("configServers");
-            for (HostName hostname : version.configServerHostnames()) {
+            for (HostName hostname : version.systemApplicationHostnames()) {
                 Cursor configServerObject = configServerArray.addObject();
                 configServerObject.setString("hostname", hostname.value());
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index c98d7b33b25..880ca98d252 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -9,12 +9,13 @@ import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.JobList;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 
-import java.net.URI;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -27,7 +28,6 @@ import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError.outOfCapacity;
 
@@ -85,12 +85,12 @@ public class VersionStatus {
 
     /** Create a full, updated version status. This is expensive and should be done infrequently */
     public static VersionStatus compute(Controller controller) {
-        ListMap<Version, HostName> configServerVersions = findConfigServerVersions(controller);
+        ListMap<Version, HostName> systemApplicationVersions = findSystemApplicationVersions(controller);
         ListMap<Version, HostName> controllerVersions = findControllerVersions(controller);
 
         Set<Version> infrastructureVersions = new HashSet<>();
         infrastructureVersions.addAll(controllerVersions.keySet());
-        infrastructureVersions.addAll(configServerVersions.keySet());
+        infrastructureVersions.addAll(systemApplicationVersions.keySet());
 
         // The controller version is the lowest controller version of all controllers
         Version controllerVersion = controllerVersions.keySet().stream().sorted().findFirst().get();
@@ -109,7 +109,7 @@ public class VersionStatus {
                 VespaVersion vespaVersion = createVersion(statistics,
                                                           statistics.version().equals(controllerVersion),
                                                           statistics.version().equals(systemVersion),
-                                                          configServerVersions.getList(statistics.version()),
+                                                          systemApplicationVersions.getList(statistics.version()),
                                                           controller);
                 versions.add(vespaVersion);
             } catch (IllegalArgumentException e) {
@@ -122,17 +122,18 @@ public class VersionStatus {
         return new VersionStatus(versions);
     }
 
-    private static ListMap<Version, HostName> findConfigServerVersions(Controller controller) {
-        List<URI> configServers = controller.zoneRegistry().zones()
-                .controllerUpgraded()
-                .ids().stream()
-                .flatMap(zoneId -> controller.zoneRegistry().getConfigServerUris(zoneId).stream())
-                .collect(Collectors.toList());
-
+    private static ListMap<Version, HostName> findSystemApplicationVersions(Controller controller) {
+        List<ZoneId> zones = controller.zoneRegistry().zones()
+                                       .controllerUpgraded()
+                                       .ids();
         ListMap<Version, HostName> versions = new ListMap<>();
-        for (URI configServer : configServers)
-            versions.put(controller.configServer().version(configServer).current(),
-                         HostName.from(configServer.getHost()));
+        for (ZoneId zone : zones) {
+            for (SystemApplication application : SystemApplication.all()) {
+                for (Node node : controller.configServer().nodeRepository().list(zone, application.id())) {
+                    versions.put(node.currentVersion(), node.hostname());
+                }
+            }
+        }
         return versions;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index 5fecb5de79f..06896412652 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -28,19 +28,19 @@ public class VespaVersion implements Comparable<VespaVersion> {
     private final boolean isControllerVersion;
     private final boolean isSystemVersion;
     private final DeploymentStatistics statistics;
-    private final ImmutableSet<HostName> configServerHostnames;
+    private final ImmutableSet<HostName> systemApplicationHostnames;
     private final Confidence confidence;
 
     public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt,
                         boolean isControllerVersion, boolean isSystemVersion,
-                        Collection<HostName> configServerHostnames,
+                        Collection<HostName> systemApplicationHostnames,
                         Confidence confidence) {
         this.statistics = statistics;
         this.releaseCommit = releaseCommit;
         this.committedAt = committedAt;
         this.isControllerVersion = isControllerVersion;
         this.isSystemVersion = isSystemVersion;
-        this.configServerHostnames = ImmutableSet.copyOf(configServerHostnames);
+        this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames);
         this.confidence = confidence;
     }
 
@@ -103,8 +103,8 @@ public class VespaVersion implements Comparable<VespaVersion> {
      */
     public boolean isSystemVersion() { return isSystemVersion; }
 
-    /** Returns the host names of the config servers (across all zones) which are currently of this version */
-    public Set<HostName> configServerHostnames() { return configServerHostnames; }
+    /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */
+    public Set<HostName> systemApplicationHostnames() { return systemApplicationHostnames; }
     
     /** Returns the confidence we have in this versions suitability for production */
     public Confidence confidence() { return confidence; }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java
index 6c7137273a7..a623f3552c0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ArtifactRepositoryMock.java
@@ -2,8 +2,10 @@
 package com.yahoo.vespa.hosted.controller;
 
 import com.yahoo.component.AbstractComponent;
+import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 
 import java.util.HashMap;
@@ -44,6 +46,11 @@ public class ArtifactRepositoryMock extends AbstractComponent implements Artifac
         return artifact.data;
     }
 
+    @Override
+    public byte[] getSystemApplicationPackage(ApplicationId application, ZoneId zone, Version version) {
+        return new byte[0];
+    }
+
     private static int artifactHash(ApplicationId applicationId, String applicationVersion) {
         return Objects.hash(applicationId, applicationVersion);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java
index 1f96fc3ef7f..9fb076880af 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerMock.java
@@ -4,9 +4,11 @@ package com.yahoo.vespa.hosted.controller;
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.node.JsonNodeFactory;
 import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
@@ -14,9 +16,11 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.vespa.serviceview.bindings.ClusterView;
 import com.yahoo.vespa.serviceview.bindings.ServiceView;
@@ -26,35 +30,57 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 /**
  * @author mortent
  */
 public class ConfigServerMock extends AbstractComponent implements ConfigServer {
 
-    private final Map<ApplicationId, Boolean> applicationActivated = new HashMap<>();
+    private final Map<ApplicationId, Application> applications = new LinkedHashMap<>();
     private final Map<String, EndpointStatus> endpoints = new HashMap<>();
     private final Map<URI, Version> versions = new HashMap<>();
-    private final Map<URI, Version> wantedVersions = new HashMap<>();
+    private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock();
 
-    private Version defaultVersion = new Version(6, 1, 0);
+    private Version initialVersion = new Version(6, 1, 0);
     private Version lastPrepareVersion = null;
     private RuntimeException prepareException = null;
 
+    @Inject
+    public ConfigServerMock(ZoneRegistryMock zoneRegistry) {
+        bootstrap(zoneRegistry.zones().all().ids());
+    }
+
+    public void bootstrap(List<ZoneId> zones) {
+        nodeRepository().clear();
+        for (ZoneId zone : zones) {
+            for (SystemApplication application : SystemApplication.all()) {
+                List<Node> nodes = IntStream.rangeClosed(1, 3)
+                                            .mapToObj(i -> new Node(
+                                                    HostName.from("node-" + i + "-" + application.id().application()
+                                                                                                 .value()),
+                                                    application.nodeType(),
+                                                    Optional.of(application.id()),
+                                                    initialVersion,
+                                                    initialVersion
+                                            ))
+                                            .collect(Collectors.toList());
+                nodeRepository().add(zone, nodes);
+            }
+        }
+    }
+
     /** The version given in the previous prepare call, or empty if no call has been made */
     public Optional<Version> lastPrepareVersion() {
         return Optional.ofNullable(lastPrepareVersion);
     }
 
-    /** Return map of applications that may have been activated */
-    public Map<ApplicationId, Boolean> activated() {
-        return Collections.unmodifiableMap(applicationActivated);
-    }
-
     /** The exception to throw on the next prepare run, or null to continue normally */
     public void throwOnNextPrepare(RuntimeException prepareException) {
         this.prepareException = prepareException;
@@ -68,34 +94,45 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return versions;
     }
 
-    @Override
-    public void upgrade(URI configServerUri, Version version) {
-        wantedVersions.put(configServerUri, version);
+    /** Set version for system applications in given zone */
+    public void setVersion(Version version, ZoneId zone, List<SystemApplication> applications) {
+        for (SystemApplication application : applications) {
+            for (Node node : nodeRepository().list(zone, application.id())) {
+                nodeRepository().add(zone, new Node(node.hostname(), node.type(), node.owner(), version, version));
+            }
+        }
+    }
+
+    /** The initial version of this config server */
+    public Version initialVersion() {
+        return initialVersion;
     }
 
-    /** Set the default config server version */
-    public void setDefaultVersion(Version version) {
-        this.defaultVersion = version;
+    /** Get deployed application by ID */
+    public Optional<Application> application(ApplicationId id) {
+        return Optional.ofNullable(applications.get(id));
     }
 
-    public Version getDefaultVersion() {
-        return defaultVersion;
+    @Override
+    public NodeRepositoryMock nodeRepository() {
+        return nodeRepository;
     }
 
     @Override
     public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames,
                                        Set<String> rotationNames, byte[] content) {
-        lastPrepareVersion = deployOptions.vespaVersion.map(Version::new).orElse(null);
+        lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null);
         if (prepareException != null) {
             RuntimeException prepareException = this.prepareException;
             this.prepareException = null;
             throw prepareException;
         }
-        applicationActivated.put(deployment.applicationId(), false);
+        applications.put(deployment.applicationId(), new Application(deployment.applicationId(), lastPrepareVersion));
 
         return new PreparedApplication() {
+
             @Override
-            public void activate() { /* Nothing to do, done in  */}
+            public void activate() {}
 
             @Override
             public List<Log> messages() {
@@ -114,7 +151,15 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
             @Override
             public PrepareResponse prepareResponse() {
-                applicationActivated.put(deployment.applicationId(), true);
+                Application application = applications.get(deployment.applicationId());
+                application.activate();
+                for (Node node : nodeRepository.list(deployment.zoneId(), deployment.applicationId())) {
+                    nodeRepository.add(deployment.zoneId(), new Node(node.hostname(),
+                                                                     node.type(),
+                                                                     node.owner(),
+                                                                     node.currentVersion(),
+                                                                     application.version().get()));
+                }
 
                 PrepareResponse prepareResponse = new PrepareResponse();
                 prepareResponse.message = "foo";
@@ -123,6 +168,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
                 prepareResponse.tenant = new TenantId("tenant");
                 return prepareResponse;
             }
+
         };
     }
 
@@ -132,7 +178,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
 
     @Override
     public void deactivate(DeploymentId deployment) {
-        applicationActivated.remove(deployment.applicationId());
+        applications.remove(deployment.applicationId());
     }
 
     @Override
@@ -176,12 +222,6 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         root.put("resources", resources);
         return root;
     }
-    
-    @Override
-    public ConfigServerVersion version(URI configServerUri) {
-        return new ConfigServerVersion(versions.getOrDefault(configServerUri, defaultVersion),
-                                       wantedVersions.getOrDefault(configServerUri, defaultVersion));
-    }
 
     @Override
     public void setGlobalRotationStatus(DeploymentId deployment, String endpoint, EndpointStatus status) {
@@ -194,4 +234,33 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return endpoints.getOrDefault(endpoint, result);
     }
 
+    public static class Application {
+
+        private final ApplicationId id;
+        private final Version version;
+        private boolean activated;
+
+        private Application(ApplicationId id, Version version) {
+            this.id = id;
+            this.version = version;
+        }
+
+        public ApplicationId id() {
+            return id;
+        }
+
+        public Optional<Version> version() {
+            return Optional.ofNullable(version);
+        }
+
+        public boolean activated() {
+            return activated;
+        }
+
+        private void activate() {
+            this.activated = true;
+        }
+
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 2042138d49b..8a15682a2ea 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -92,7 +92,7 @@ public class ControllerTest {
                 .build();
 
         // staging job - succeeding
-        Version version1 = tester.defaultVespaVersion();
+        Version version1 = tester.defaultPlatformVersion();
         Application app1 = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.jobCompletion(component).application(app1).uploadArtifact(applicationPackage).submit();
         assertEquals("Application version is known from completion of initial job",
@@ -309,7 +309,7 @@ public class ControllerTest {
                 versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.committedAt(),
                                                  false,
                                                  false,
-                                                 c.configServerHostnames(),
+                                                 c.systemApplicationHostnames(),
                                                  c.confidence()));
         }
         versions.add(newSystemVespaVersion);
@@ -774,7 +774,7 @@ public class ControllerTest {
         tester.controller().applications().deploy(app.id(), zone, Optional.of(applicationPackage), options);
 
         assertTrue("Application deployed and activated",
-                   tester.controllerTester().configServer().activated().getOrDefault(app.id(), false));
+                   tester.controllerTester().configServer().application(app.id()).get().activated());
 
         assertTrue("No job status added",
                    tester.applications().require(app.id()).deploymentJobs().jobStatus().isEmpty());
@@ -806,7 +806,7 @@ public class ControllerTest {
 
     private void runDeployment(DeploymentTester tester, Application app, ApplicationVersion version,
                                Optional<Version> upgrade, Optional<ApplicationPackage> applicationPackage) {
-        Version vespaVersion = upgrade.orElseGet(tester::defaultVespaVersion);
+        Version vespaVersion = upgrade.orElseGet(tester::defaultPlatformVersion);
 
         // Deploy in test
         tester.deployAndNotify(app, applicationPackage, true, systemTest);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 271ed4627b4..52225a4c62e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -73,7 +73,7 @@ public final class ControllerTester {
     private Controller controller;
 
     public ControllerTester(ManualClock clock, RotationsConfig rotationsConfig, MockCuratorDb curatorDb) {
-        this(new AthenzDbMock(), clock, new ConfigServerMock(),
+        this(new AthenzDbMock(), clock, new ConfigServerMock(new ZoneRegistryMock()),
              new ZoneRegistryMock(), new GitHubMock(), curatorDb, rotationsConfig,
              new MemoryNameService(), new ArtifactRepositoryMock(), new MemoryEntityService(), new MockBuildService());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/NodeRepositoryMock.java
new file mode 100644
index 00000000000..1c42fb3501c
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/NodeRepositoryMock.java
@@ -0,0 +1,62 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.NodeType;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * @author mpolden
+ */
+public class NodeRepositoryMock implements NodeRepository {
+
+    private final Map<ZoneId, Map<HostName, Node>> nodeRepository = new HashMap<>();
+
+    public void add(ZoneId zone, List<Node> nodes) {
+        nodeRepository.compute(zone, (k, v) -> {
+            if (v == null) {
+                v = new HashMap<>();
+            }
+            for (Node node : nodes) {
+                v.put(node.hostname(), node);
+            }
+            return v;
+        });
+    }
+
+    public void add(ZoneId zone, Node node) {
+        add(zone, Collections.singletonList(node));
+    }
+
+    public void clear() {
+        nodeRepository.clear();
+    }
+
+    @Override
+    public List<Node> list(ZoneId zone, ApplicationId application) {
+        return nodeRepository.getOrDefault(zone, Collections.emptyMap()).values().stream()
+                             .filter(node -> node.owner().map(application::equals).orElse(false))
+                             .collect(Collectors.toList());
+    }
+
+    @Override
+    public void upgrade(ZoneId zone, NodeType type, Version version) {
+        nodeRepository.getOrDefault(zone, Collections.emptyMap()).values()
+                      .stream()
+                      .filter(node -> node.type() == type)
+                      .map(node -> new Node(node.hostname(), node.type(), node.owner(), node.currentVersion(),
+                                            version))
+                      .forEach(node -> add(zone, node));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 8e671e4ebaf..ffc93f7b5ba 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -14,10 +14,12 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.ReadyJobsTrigger;
 import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader;
@@ -96,21 +98,29 @@ public class DeploymentTester {
         return controller().applications().require(application);
     }
 
+    /** Re-compute and write version status */
     public void computeVersionStatus() {
         controller().updateVersionStatus(VersionStatus.compute(controller()));
     }
 
     /** Upgrade controller to given version */
-    public void upgradeController(Version newVersion) {
-        controller().curator().writeControllerVersion(controller().hostname(), newVersion);
-        controller().updateVersionStatus(VersionStatus.compute(controller()));
+    public void upgradeController(Version version) {
+        controller().curator().writeControllerVersion(controller().hostname(), version);
+        computeVersionStatus();
+    }
+
+    /** Upgrade system applications in all zones to given version */
+    public void upgradeSystemApplications(Version version) {
+        for (ZoneId zone : tester.zoneRegistry().zones().all().ids()) {
+            tester.configServer().setVersion(version, zone, SystemApplication.all());
+        }
+        computeVersionStatus();
     }
 
     /** Upgrade entire system to given version */
     public void upgradeSystem(Version version) {
         upgradeController(version);
-        configServer().setDefaultVersion(version);
-        computeVersionStatus();
+        upgradeSystemApplications(version);
         upgrader().maintain();
         readyJobTrigger().maintain();
     }
@@ -119,8 +129,8 @@ public class DeploymentTester {
         while (deploymentTrigger().triggerReadyJobs() > 0);
     }
 
-    public Version defaultVespaVersion() {
-        return configServer().getDefaultVersion();
+    public Version defaultPlatformVersion() {
+        return configServer().initialVersion();
     }
 
     public Application createApplication(String applicationName, String tenantName, long projectId, Long propertyId) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 306244f5b80..991448e8fca 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -2,31 +2,34 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.NodeRepositoryMock;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import org.junit.Test;
 
-import java.net.URI;
 import java.util.Arrays;
+import java.util.function.Function;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
 
 /**
  * @author mpolden
  */
 public class SystemUpgraderTest {
 
+    private static final ZoneId zone1 = ZoneId.from("prod", "eu-west-1");
+    private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1");
+    private static final ZoneId zone3 = ZoneId.from("prod", "us-central-1");
+    private static final ZoneId zone4 = ZoneId.from("prod", "us-east-3");
+
     private final DeploymentTester tester = new DeploymentTester();
 
     @Test
     public void upgrade_system() {
-        ZoneId zone1 = ZoneId.from("prod", "eu-west-1");
-        ZoneId zone2 = ZoneId.from("prod", "us-west-1");
-        ZoneId zone3 = ZoneId.from("prod", "us-central-1");
-        ZoneId zone4 = ZoneId.from("prod", "us-east-3");
-
         tester.controllerTester().zoneRegistry().setUpgradePolicy(
                 UpgradePolicy.create()
                              .upgrade(zone1)
@@ -35,38 +38,70 @@ public class SystemUpgraderTest {
         );
 
         Version version1 = Version.fromString("6.5");
+        tester.configServer().bootstrap(Arrays.asList(zone1, zone2, zone3, zone4));
         tester.upgradeSystem(version1);
         tester.systemUpgrader().maintain();
-        assertTrue("Zones are on current version", onVersion(version1, zone1, zone2, zone3, zone4));
+        assertCurrentVersion(SystemApplication.configServer, version1, zone1, zone2, zone3, zone4);
+        assertCurrentVersion(SystemApplication.zone, version1, zone1, zone2, zone3, zone4);
 
         // Controller upgrades
         Version version2 = Version.fromString("6.6");
         tester.upgradeController(version2);
         assertEquals(version2, tester.controller().versionStatus().controllerVersion().get().versionNumber());
 
-        // System upgrade scheduled. First zone starts upgrading
+        // System upgrade starts
+        tester.systemUpgrader().maintain();
+        assertWantedVersion(SystemApplication.configServer, version2, zone1);
+        // Other zones remain on previous version
+        assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4);
+        // Zone application is not upgraded yet
+        assertWantedVersion(SystemApplication.zone, version1, zone1, zone2, zone3, zone4);
+
+        // zone1: zone-config-server upgrades
+        completeUpgrade(SystemApplication.configServer, version2, zone1);
+
+        // zone 1: zone-application upgrades
+        tester.systemUpgrader().maintain();
+        assertWantedVersion(SystemApplication.zone, version2, zone1);
+        completeUpgrade(SystemApplication.zone, version2, zone1);
+
+        // zone 2, 3 and 4: still targets old version
+        assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4);
+        assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4);
+
+        // zone 2 and 3: zone-config-server upgrades in parallel
         tester.systemUpgrader().maintain();
-        assertWantedVersion(version2, zone1);
-        assertWantedVersion(version1, zone2, zone3, zone4); // Other zones remains on previous version
+        assertWantedVersion(SystemApplication.configServer, version2, zone2, zone3);
+        assertWantedVersion(SystemApplication.configServer, version1, zone4);
+        assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4);
+        completeUpgrade(SystemApplication.configServer, version2, zone2, zone3);
 
-        // Zones 1 completes upgrade
-        completeUpgrade(version2, zone1);
+        // zone 2 and 3: zone-application upgrades in parallel
+        tester.systemUpgrader().maintain();
+        assertWantedVersion(SystemApplication.zone, version2, zone2, zone3);
+        completeUpgrade(SystemApplication.zone, version2, zone2, zone3);
 
-        // Zones 2 and 3 upgrade in parallel
+        // zone 4: zone-config-server upgrades
         tester.systemUpgrader().maintain();
-        assertWantedVersion(version2, zone2, zone3);
-        assertWantedVersion(version1, zone4);
-        completeUpgrade(version2, zone2, zone3);
+        assertWantedVersion(SystemApplication.configServer, version2, zone4);
+        assertWantedVersion(SystemApplication.zone, version1, zone4);
+        completeUpgrade(SystemApplication.configServer, version2, zone4);
 
-        // Zone 4 upgrades last
+        // System version remains unchanged until final application upgrades
+        tester.computeVersionStatus();
+        assertEquals(version1, tester.controller().versionStatus().systemVersion().get().versionNumber());
+
+        // zone 4: zone-application upgrades
         tester.systemUpgrader().maintain();
-        assertWantedVersion(version2, zone4);
-        completeUpgrade(version2, zone4);
+        assertWantedVersion(SystemApplication.zone, version2, zone4);
+        completeUpgrade(SystemApplication.zone, version2, zone4);
+        tester.computeVersionStatus();
+        assertEquals(version2, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
         // Next run does nothing as system is now upgraded
         tester.systemUpgrader().maintain();
-        assertWantedVersion(version2, zone1, zone2, zone3, zone4);
-        assertTrue(onVersion(version2, zone1, zone2, zone3, zone4));
+        assertWantedVersion(SystemApplication.configServer, version2, zone1, zone2, zone3, zone4);
+        assertWantedVersion(SystemApplication.zone, version2, zone1, zone2, zone3, zone4);
     }
 
     @Test
@@ -77,37 +112,48 @@ public class SystemUpgraderTest {
         Version version = Version.fromString("6.5");
         tester.upgradeSystem(version);
         tester.systemUpgrader().maintain();
-        assertTrue("Zone is on current version", onVersion(version, zone));
+        assertWantedVersion(SystemApplication.configServer, version, zone);
+        assertWantedVersion(SystemApplication.zone, version, zone);
 
         // Controller is downgraded
         tester.upgradeController(Version.fromString("6.4"));
 
         // Wanted version for zone remains unchanged
         tester.systemUpgrader().maintain();
-        assertWantedVersion(version, zone);
+        assertWantedVersion(SystemApplication.configServer, version, zone);
+        assertWantedVersion(SystemApplication.zone, version, zone);
     }
 
-    private void completeUpgrade(Version version, ZoneId... zones) {
+    /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */
+    private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) {
         for (ZoneId zone : zones) {
-            for (URI configServer : tester.controller().zoneRegistry().getConfigServerUris(zone)) {
-                tester.controllerTester().configServer().versions().put(configServer, version);
+            for (Node node : nodeRepository().list(zone, application.id())) {
+                nodeRepository().add(zone, new Node(node.hostname(), node.type(), node.owner(), node.wantedVersion(),
+                                                    node.wantedVersion()));
             }
-            assertTrue(onVersion(version, zone));
+            assertCurrentVersion(application, version, zone);
         }
     }
 
-    private void assertWantedVersion(Version version, ZoneId... zones) {
+    private void assertWantedVersion(SystemApplication application, Version version, ZoneId... zones) {
+        assertVersion(application.id(), version, Node::wantedVersion, zones);
+    }
+
+    private void assertCurrentVersion(SystemApplication application, Version version, ZoneId... zones) {
+        assertVersion(application.id(), version, Node::currentVersion, zones);
+    }
+
+    private void assertVersion(ApplicationId applicationId, Version version, Function<Node, Version> versionField,
+                               ZoneId... zones) {
         for (ZoneId zone : zones) {
-            for (URI configServer : tester.controller().zoneRegistry().getConfigServerUris(zone)) {
-                assertEquals(version, tester.controller().configServer().version(configServer).wanted());
+            for (Node node : nodeRepository().list(zone, applicationId)) {
+                assertEquals(version, versionField.apply(node));
             }
         }
     }
 
-    private boolean onVersion(Version version, ZoneId... zone) {
-        return Arrays.stream(zone)
-                     .flatMap(z -> tester.controller().zoneRegistry().getConfigServerUris(z).stream())
-                     .allMatch(uri -> tester.controller().configServer().version(uri).current().equals(version));
+    private NodeRepositoryMock nodeRepository() {
+        return tester.controllerTester().configServer().nodeRepository();
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
index b0306178ae3..bba53f42a61 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
@@ -50,7 +50,7 @@ public class VersionStatusSerializerTest {
             assertEquals(a.isControllerVersion(), b.isControllerVersion());
             assertEquals(a.isSystemVersion(), b.isSystemVersion());
             assertEquals(a.statistics(), b.statistics());
-            assertEquals(a.configServerHostnames(), b.configServerHostnames());
+            assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames());
             assertEquals(a.confidence(), b.confidence());
         }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
index 2bdc6a493c2..c72edf7e403 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
@@ -12,6 +12,8 @@ import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
 import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain;
 import com.yahoo.vespa.hosted.controller.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import org.junit.ComparisonFailure;
 
@@ -54,7 +56,9 @@ public class ContainerTester {
 
     public void upgradeSystem(Version version) {
         controller().curator().writeControllerVersion(controller().hostname(), version);
-        configServer().setDefaultVersion(version);
+        for (ZoneId zone : controller().zoneRegistry().zones().all().ids()) {
+            configServer().setVersion(version, zone, SystemApplication.all());
+        }
         computeVersionStatus();
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index 9e854c8bf03..fa93c0f1df1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -83,7 +83,7 @@ public class DeploymentApiTest extends ControllerContainerTest {
     private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) {
         List<VespaVersion> censored = new ArrayList<>();
         for (VespaVersion version : versionStatus.versions()) {
-            if ( ! version.configServerHostnames().isEmpty())
+            if (!version.systemApplicationHostnames().isEmpty()) {
                 version = new VespaVersion(version.statistics(),
                                            version.releaseCommit(),
                                            version.committedAt(),
@@ -94,6 +94,7 @@ public class DeploymentApiTest extends ControllerContainerTest {
                                                        .collect(Collectors.toSet()),
                                            VespaVersion.confidenceFrom(version.statistics(), controller)
                 );
+            }
             censored.add(version);
         }
         return new VersionStatus(censored);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 6ab998d64cb..9450b6223b1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -10,14 +10,16 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
 import org.junit.Test;
 
-import java.net.URI;
 import java.util.List;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -46,9 +48,9 @@ public class VersionStatusTest {
 
     @Test
     public void testSystemVersionIsControllerVersionIfConfigServersAreNewer() {
-        ControllerTester tester = new ControllerTester();
+        DeploymentTester tester = new DeploymentTester();
         Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1);
-        tester.configServer().setDefaultVersion(largerThanCurrent);
+        tester.upgradeSystemApplications(largerThanCurrent);
         VersionStatus versionStatus = VersionStatus.compute(tester.controller());
         assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber());
     }
@@ -57,7 +59,13 @@ public class VersionStatusTest {
     public void testSystemVersionIsVersionOfOldestConfigServer() {
         ControllerTester tester = new ControllerTester();
         Version oldest = new Version(5);
-        tester.configServer().versions().put(URI.create("https://cfg.prod.corp-us-east-1.test:4443"), oldest);
+        for (ZoneId zone : tester.zoneRegistry().zones().all().ids()) {
+            for (Node node : tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id())) {
+                tester.configServer().nodeRepository().add(zone, new Node(node.hostname(), node.type(), node.owner(),
+                                                                          oldest, node.wantedVersion()));
+                break;
+            }
+        }
         VersionStatus versionStatus = VersionStatus.compute(tester.controller());
         assertEquals(oldest, versionStatus.systemVersion().get().versionNumber());
     }