Emit metric for nodes failing system upgrade

18 files changed, 494 insertions, 152 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
index 9df918e3f20..5ff564f7ad3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java
@@ -63,7 +63,7 @@ public enum SystemApplication {
                          .orElse(false);
     }
 
-    /** Returns the node types of this that should receive OS upgrades */
+    /** Returns whether this should receive OS upgrades */
     public boolean isEligibleForOsUpgrades() {
         return nodeType.isDockerHost();
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 79ababd20d3..650b8400298 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.google.common.collect.ImmutableMap;
@@ -18,7 +18,6 @@ import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.stream.Collectors;
@@ -34,9 +33,12 @@ public class MetricsReporter extends Maintainer {
     public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades";
     public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds";
     public static final String DEPLOYMENT_WARNINGS = "deployment.warnings";
+    public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade";
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
 
+    private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofHours(1);
+
     private final Metric metric;
     private final Clock clock;
 
@@ -51,12 +53,13 @@ public class MetricsReporter extends Maintainer {
         reportDeploymentMetrics();
         reportRemainingRotations();
         reportQueuedNameServiceRequests();
+        reportNodesFailingSystemUpgrade();
     }
 
     private void reportRemainingRotations() {
         try (RotationLock lock = controller().applications().rotationRepository().lock()) {
             int availableRotations = controller().applications().rotationRepository().availableRotations(lock).size();
-            metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Collections.emptyMap()));
+            metric.set(REMAINING_ROTATIONS, availableRotations, metric.createContext(Map.of()));
         }
     }
 
@@ -66,7 +69,7 @@ public class MetricsReporter extends Maintainer {
                                                   .flatMap(application -> application.instances().values().stream())
                                                   .collect(Collectors.toUnmodifiableList());
 
-        metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(instances) * 100, metric.createContext(Collections.emptyMap()));
+        metric.set(DEPLOYMENT_FAIL_METRIC, deploymentFailRatio(instances) * 100, metric.createContext(Map.of()));
 
         averageDeploymentDurations(instances, clock.instant()).forEach((application, duration) -> {
             metric.set(DEPLOYMENT_AVERAGE_DURATION, duration.getSeconds(), metric.createContext(dimensions(application)));
@@ -93,6 +96,20 @@ public class MetricsReporter extends Maintainer {
         metric.set(NAME_SERVICE_REQUESTS_QUEUED, controller().curator().readNameServiceQueue().requests().size(),
                    metric.createContext(Map.of()));
     }
+
+    private void reportNodesFailingSystemUpgrade() {
+        metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of()));
+    }
+
+    private int nodesFailingSystemUpgrade() {
+        if (!controller().versionStatus().isUpgrading()) return 0;
+        var nodesFailingUpgrade = 0;
+        var timeoutInstant = clock.instant().minus(NODE_UPGRADE_TIMEOUT);
+        for (var vespaVersion : controller().versionStatus().versions()) {
+            nodesFailingUpgrade += vespaVersion.nodeVersions().changedBefore(timeoutInstant).size();
+        }
+        return nodesFailingUpgrade;
+    }
     
     private static double deploymentFailRatio(List<Instance> instances) {
         return instances.stream()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
index 207a5f8dcf9..bb74b26bf93 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
@@ -9,6 +9,8 @@ import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
 import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
@@ -16,6 +18,7 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Set;
@@ -47,6 +50,13 @@ public class VersionStatusSerializer {
     private static final String confidenceField = "confidence";
     private static final String configServersField = "configServerHostnames";
 
+    // NodeVersions fields
+    private static final String nodeVersionsField = "nodeVersions";
+
+    // NodeVersion fields
+    private static final String hostnameField = "hostname";
+    private static final String changedAtField = "changedAt";
+
     // DeploymentStatistics fields
     private static final String versionField = "version";
     private static final String failingField = "failing";
@@ -77,9 +87,20 @@ public class VersionStatusSerializer {
         object.setBool(isReleasedField, version.isReleased());
         deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField));
         object.setString(confidenceField, version.confidence().name());
-        configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField));
+        configServersToSlime(version.nodeVersions().hostnames(), object.setArray(configServersField));
+        nodeVersionsToSlime(version.nodeVersions(), object.setArray(nodeVersionsField));
+    }
+
+    private void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) {
+        for (NodeVersion nodeVersion : nodeVersions.asMap().values()) {
+            var nodeVersionObject = array.addObject();
+            nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value());
+            nodeVersionObject.setString(versionField, nodeVersion.version().toFullString());
+            nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli());
+        }
     }
 
+    // TODO(mpolden): Remove after October 2019
     private void configServersToSlime(Set<HostName> configServerHostnames, Cursor array) {
         configServerHostnames.stream().map(HostName::value).forEach(array::addString);
     }
@@ -107,12 +128,32 @@ public class VersionStatusSerializer {
                                 Instant.ofEpochMilli(object.field(committedAtField).asLong()),
                                 object.field(isControllerVersionField).asBool(),
                                 object.field(isSystemVersionField).asBool(),
-                                object.field(isReleasedField).valid() ? object.field(isReleasedField).asBool() : true,
-                                configServersFromSlime(object.field(configServersField)),
+                                !object.field(isReleasedField).valid() || object.field(isReleasedField).asBool(),
+                                nodeVersionsFromSlime(object),
                                 VespaVersion.Confidence.valueOf(object.field(confidenceField).asString())
         );
     }
 
+    private NodeVersions nodeVersionsFromSlime(Inspector root) {
+        var nodeVersions = new LinkedHashMap<HostName, NodeVersion>();
+        var nodeVersionsRoot = root.field(nodeVersionsField);
+        if (nodeVersionsRoot.valid()) {
+            nodeVersionsRoot.traverse((ArrayTraverser) (i, entry) -> {
+                var hostname = HostName.from(entry.field(hostnameField).asString());
+                var version = Version.fromString(entry.field(versionField).asString());
+                var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong());
+                nodeVersions.put(hostname, new NodeVersion(hostname, version, changedAt));
+            });
+        } else {
+            // TODO(mpolden): Remove after October 2019
+            var configServerHostnames = configServersFromSlime(root.field(configServersField));
+            for (var hostname : configServerHostnames) {
+                nodeVersions.put(hostname, NodeVersion.empty(hostname));
+            }
+        }
+        return new NodeVersions(nodeVersions);
+    }
+
     private Set<HostName> configServersFromSlime(Inspector array) {
         Set<HostName> configServerHostnames = new LinkedHashSet<>();
         array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(HostName.from(entry.asString())));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
index 86310ca2f6b..2adf6ce95e1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
 import com.yahoo.component.Version;
@@ -91,7 +91,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler {
             versionObject.setBool("systemVersion", version.isSystemVersion());
 
             Cursor configServerArray = versionObject.setArray("configServers");
-            for (HostName hostname : version.systemApplicationHostnames()) {
+            for (HostName hostname : version.nodeVersions().hostnames()) {
                 Cursor configServerObject = configServerArray.addObject();
                 configServerObject.setString("hostname", hostname.value());
             }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
new file mode 100644
index 00000000000..3ba1e4800db
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
@@ -0,0 +1,68 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostName;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}.
+ *
+ * This is immutable.
+ *
+ * @author mpolden
+ */
+public class NodeVersion {
+
+    private final HostName hostname;
+    private final Version version;
+    private final Instant changedAt;
+
+    public NodeVersion(HostName hostname, Version version, Instant changedAt) {
+        this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null");
+        this.version = Objects.requireNonNull(version, "version must be non-null");
+        this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null");
+    }
+
+    /** Hostname of this */
+    public HostName hostname() {
+        return hostname;
+    }
+
+    /** Current version of this */
+    public Version version() {
+        return version;
+    }
+
+    /** The most recent time the version of this changed */
+    public Instant changedAt() {
+        return changedAt;
+    }
+
+    @Override
+    public String toString() {
+        return hostname + " on " + version + " [changed at " + changedAt + "]";
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodeVersion that = (NodeVersion) o;
+        return hostname.equals(that.hostname) &&
+               version.equals(that.version) &&
+               changedAt.equals(that.changedAt);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(hostname, version, changedAt);
+    }
+
+    public static NodeVersion empty(HostName hostname) {
+        return new NodeVersion(hostname, Version.emptyVersion, Instant.EPOCH);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
new file mode 100644
index 00000000000..80ecc2dd9bb
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
@@ -0,0 +1,107 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.versions;
+
+import com.google.common.collect.ImmutableListMultimap;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.ListMultimap;
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.HostName;
+
+import java.time.Instant;
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
+
+import static java.util.stream.Collectors.collectingAndThen;
+import static java.util.stream.Collectors.toMap;
+
+/**
+ * A filterable list of {@link NodeVersion}s. This is immutable.
+ *
+ * @author mpolden
+ */
+public class NodeVersions {
+
+    public static final NodeVersions EMPTY = new NodeVersions(Map.of());
+
+    private final Map<HostName, NodeVersion> nodeVersions;
+
+    public NodeVersions(Map<HostName, NodeVersion> nodeVersions) {
+        this.nodeVersions = ImmutableSortedMap.copyOf(Objects.requireNonNull(nodeVersions));
+    }
+
+    public Map<HostName, NodeVersion> asMap() {
+        return nodeVersions;
+    }
+
+    /** Returns host names in this, grouped by version */
+    public ListMultimap<Version, HostName> asVersionMap() {
+        var versions = ImmutableListMultimap.<Version, HostName>builder();
+        for (var kv : nodeVersions.entrySet()) {
+            versions.put(kv.getValue().version(), kv.getKey());
+        }
+        return versions.build();
+    }
+
+    /** Returns host names in this */
+    public Set<HostName> hostnames() {
+        return nodeVersions.keySet();
+    }
+
+    /** Returns a copy of this containing only node versions of given version */
+    public NodeVersions matching(Version version) {
+        return filter(nodeVersion -> nodeVersion.version().equals(version));
+    }
+
+    /** Returns a copy of this containing only node versions that last changed before given instant */
+    public NodeVersions changedBefore(Instant instant) {
+        return filter(nodeVersion -> nodeVersion.changedAt().isBefore(instant));
+    }
+
+    /** Returns a copy of this retaining only node versions for the given host names */
+    public NodeVersions retainAll(Set<HostName> hostnames) {
+        if (this.nodeVersions.keySet().equals(hostnames)) return this;
+
+        var nodeVersions = new LinkedHashMap<>(this.nodeVersions);
+        nodeVersions.keySet().retainAll(hostnames);
+        return new NodeVersions(nodeVersions);
+    }
+
+    /** Returns number of node versions in this */
+    public int size() {
+        return nodeVersions.size();
+    }
+
+    /** Returns a copy of this with a new node version added. Duplicate node versions are ignored */
+    public NodeVersions with(NodeVersion nodeVersion) {
+        var existing = nodeVersions.get(nodeVersion.hostname());
+        if (existing != null && existing.version().equals(nodeVersion.version())) return this;
+
+        var nodeVersions = new LinkedHashMap<>(this.nodeVersions);
+        nodeVersions.put(nodeVersion.hostname(), nodeVersion);
+        return new NodeVersions(nodeVersions);
+    }
+
+    private NodeVersions filter(Predicate<NodeVersion> predicate) {
+        return nodeVersions.entrySet().stream()
+                           .filter(kv -> predicate.test(kv.getValue()))
+                           .collect(collectingAndThen(toMap(Map.Entry::getKey, Map.Entry::getValue),
+                                                      NodeVersions::new));
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        NodeVersions that = (NodeVersions) o;
+        return nodeVersions.equals(that.nodeVersions);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(nodeVersions);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 9dc6b86e4be..8c7e539dee4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -6,12 +6,10 @@ import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ListMultimap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
-import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.JobList;
@@ -23,6 +21,7 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -70,7 +69,8 @@ public class VersionStatus {
     /** Returns whether the system is currently upgrading */
     public boolean isUpgrading() {
         return systemVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion)
-                              .isBefore(controllerVersion().map(VespaVersion::versionNumber).orElse(Version.emptyVersion));
+                              .isBefore(controllerVersion().map(VespaVersion::versionNumber)
+                                                           .orElse(Version.emptyVersion));
     }
 
     /** 
@@ -91,14 +91,14 @@ public class VersionStatus {
 
     /** Create a full, updated version status. This is expensive and should be done infrequently */
     public static VersionStatus compute(Controller controller) {
-        ListMultimap<Version, HostName> systemApplicationVersions = findSystemApplicationVersions(controller);
-        ListMultimap<ControllerVersion, HostName> controllerVersions = findControllerVersions(controller);
+        var systemApplicationVersions = findSystemApplicationVersions(controller);
+        var controllerVersions = findControllerVersions(controller);
 
-        ListMultimap<Version, HostName> infrastructureVersions = ArrayListMultimap.create();
+        var infrastructureVersions = ArrayListMultimap.<Version, HostName>create();
         for (var kv : controllerVersions.asMap().entrySet()) {
             infrastructureVersions.putAll(kv.getKey().version(), kv.getValue());
         }
-        infrastructureVersions.putAll(systemApplicationVersions);
+        infrastructureVersions.putAll(systemApplicationVersions.asVersionMap());
 
         // The controller version is the lowest controller version of all controllers
         ControllerVersion controllerVersion = controllerVersions.keySet().stream()
@@ -138,7 +138,7 @@ public class VersionStatus {
                                                           controllerVersion,
                                                           systemVersion,
                                                           isReleased,
-                                                          systemApplicationVersions.get(statistics.version()),
+                                                          systemApplicationVersions.matching(statistics.version()),
                                                           controller);
                 versions.add(vespaVersion);
             } catch (IllegalArgumentException e) {
@@ -152,29 +152,33 @@ public class VersionStatus {
         return new VersionStatus(versions);
     }
 
-    private static ListMultimap<Version, HostName> findSystemApplicationVersions(Controller controller) {
-        ListMultimap<Version, HostName> versions = ArrayListMultimap.create();
-        for (ZoneApi zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) {
-            for (SystemApplication application : SystemApplication.all()) {
-                List<Node> eligibleForUpgradeApplicationNodes = controller.serviceRegistry().configServer().nodeRepository()
-                        .list(zone.getId(), application.id()).stream()
-                        .filter(SystemUpgrader::eligibleForUpgrade)
-                        .collect(Collectors.toList());
-                if (eligibleForUpgradeApplicationNodes.isEmpty())
-                    continue;
-
-                boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty());
+    private static NodeVersions findSystemApplicationVersions(Controller controller) {
+        var nodeVersions = controller.versionStatus().systemVersion()
+                                     .map(VespaVersion::nodeVersions)
+                                     .orElse(NodeVersions.EMPTY);
+        var hostnames = new HashSet<HostName>();
+        for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) {
+            for (var application : SystemApplication.all()) {
+                var nodes = controller.serviceRegistry().configServer().nodeRepository()
+                                      .list(zone.getId(), application.id()).stream()
+                                      .filter(SystemUpgrader::eligibleForUpgrade)
+                                      .collect(Collectors.toList());
+                if (nodes.isEmpty()) continue;
+                var configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty());
                 if (!configConverged) {
-                    log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() + " has not converged");
+                    log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() +
+                                              " has not converged");
                 }
-                for (Node node : eligibleForUpgradeApplicationNodes) {
+                var now = controller.clock().instant();
+                for (var node : nodes) {
                     // Only use current node version if config has converged
-                    Version nodeVersion = configConverged ? node.currentVersion() : controller.systemVersion();
-                    versions.put(nodeVersion, node.hostname());
+                    Version version = configConverged ? node.currentVersion() : controller.systemVersion();
+                    nodeVersions = nodeVersions.with(new NodeVersion(node.hostname(), version, now));
+                    hostnames.add(node.hostname());
                 }
             }
         }
-        return versions;
+        return nodeVersions.retainAll(hostnames);
     }
 
     private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) {
@@ -241,7 +245,7 @@ public class VersionStatus {
                                               ControllerVersion controllerVersion,
                                               Version systemVersion,
                                               boolean isReleased,
-                                              Collection<HostName> configServerHostnames,
+                                              NodeVersions nodeVersions,
                                               Controller controller) {
         var isSystemVersion = statistics.version().equals(systemVersion);
         var isControllerVersion = statistics.version().equals(controllerVersion.version());
@@ -279,7 +283,7 @@ public class VersionStatus {
                                 isControllerVersion,
                                 isSystemVersion,
                                 isReleased,
-                                configServerHostnames,
+                                nodeVersions,
                                 confidence);
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index dc0b2c12d5c..0d144913022 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -1,16 +1,12 @@
 // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
-import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
-import java.util.Collection;
-import java.util.Set;
 
 import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
 
@@ -30,12 +26,12 @@ public class VespaVersion implements Comparable<VespaVersion> {
     private final boolean isSystemVersion;
     private final boolean isReleased;
     private final DeploymentStatistics statistics;
-    private final ImmutableSet<HostName> systemApplicationHostnames;
+    private final NodeVersions nodeVersions;
     private final Confidence confidence;
 
     public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt,
                         boolean isControllerVersion, boolean isSystemVersion, boolean isReleased,
-                        Collection<HostName> systemApplicationHostnames,
+                        NodeVersions nodeVersions,
                         Confidence confidence) {
         this.statistics = statistics;
         this.releaseCommit = releaseCommit;
@@ -43,7 +39,7 @@ public class VespaVersion implements Comparable<VespaVersion> {
         this.isControllerVersion = isControllerVersion;
         this.isSystemVersion = isSystemVersion;
         this.isReleased = isReleased;
-        this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames);
+        this.nodeVersions = nodeVersions;
         this.confidence = confidence;
     }
 
@@ -108,9 +104,11 @@ public class VespaVersion implements Comparable<VespaVersion> {
     /** Returns whether the artifacts of this release are available in the configured maven repository. */
     public boolean isReleased() { return isReleased; }
 
-    /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */
-    public Set<HostName> systemApplicationHostnames() { return systemApplicationHostnames; }
-    
+    /** Returns the versions of nodes allocated to system applications (across all zones) */
+    public NodeVersions nodeVersions() {
+        return nodeVersions;
+    }
+
     /** Returns the confidence we have in this versions suitability for production */
     public Confidence confidence() { return confidence; }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 5dc6fb183a2..61b393efbff 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -148,8 +148,13 @@ public class DeploymentTester {
 
     /** Upgrade system applications in all zones to given version */
     public void upgradeSystemApplications(Version version) {
+        upgradeSystemApplications(version, SystemApplication.all());
+    }
+
+    /** Upgrade given system applications in all zones to version */
+    public void upgradeSystemApplications(Version version, List<SystemApplication> systemApplications) {
         for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) {
-            for (SystemApplication application : SystemApplication.all()) {
+            for (SystemApplication application : systemApplications) {
                 tester.configServer().setVersion(application.id(), zone.getId(), version);
                 tester.configServer().convergeServices(application.id(), zone.getId());
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 4a7ee8bcb63..6d2090b652b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
 import com.google.inject.Inject;
@@ -110,7 +110,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
                 List<Node> nodes = IntStream.rangeClosed(1, 3)
                                             .mapToObj(i -> new Node(
                                                     HostName.from("node-" + i + "-" + application.id().application()
-                                                                                                 .value()),
+                                                                                                 .value()
+                                                                  + "-" + zone.value()),
                                                     Node.State.active, application.nodeType(),
                                                     Optional.of(application.id()),
                                                     initialVersion,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index 4fc952b0b15..f285e771a23 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -1,4 +1,4 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
@@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester;
@@ -19,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Test;
 
 import java.time.Duration;
+import java.util.List;
 import java.util.Optional;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component;
@@ -213,6 +215,43 @@ public class MetricsReporterTest {
         assertEquals("Queue consumed", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue());
     }
 
+    @Test
+    public void test_nodes_failing_system_upgrade() {
+        var tester = new DeploymentTester();
+        var reporter = createReporter(tester.controller());
+
+        // System on initial version
+        var version0 = Version.fromString("7.0");
+        tester.upgradeSystem(version0);
+        reporter.maintain();
+        assertEquals(0, getNodesFailingUpgrade());
+
+        // System starts upgrading to next version
+        var version1 = Version.fromString("7.1");
+        tester.upgradeController(version1);
+        reporter.maintain();
+        assertEquals(0, getNodesFailingUpgrade());
+
+        // Some time passes, but only a subset of nodes upgrade within timeout
+        tester.clock().advance(Duration.ofMinutes(30));
+        tester.upgradeSystemApplications(version1, List.of(SystemApplication.configServerHost));
+        reporter.maintain();
+        assertEquals(0, getNodesFailingUpgrade());
+        tester.clock().advance(Duration.ofMinutes(30).plus(Duration.ofSeconds(1)));
+        reporter.maintain();
+        assertEquals(48, getNodesFailingUpgrade());
+
+        // Some nodes are repaired and upgrade
+        tester.upgradeSystemApplications(version1, List.of(SystemApplication.configServer));
+        reporter.maintain();
+        assertEquals(36, getNodesFailingUpgrade());
+
+        // All nodes are repaired and system upgrades
+        tester.upgradeSystem(version1);
+        reporter.maintain();
+        assertEquals(0, getNodesFailingUpgrade());
+    }
+
     private Duration getAverageDeploymentDuration(ApplicationId id) {
         return Duration.ofSeconds(getMetric(MetricsReporter.DEPLOYMENT_AVERAGE_DURATION, id).longValue());
     }
@@ -225,6 +264,10 @@ public class MetricsReporterTest {
         return getMetric(MetricsReporter.DEPLOYMENT_WARNINGS, id).intValue();
     }
 
+    private int getNodesFailingUpgrade() {
+        return metrics.getMetric(MetricsReporter.NODES_FAILING_SYSTEM_UPGRADE).intValue();
+    }
+
     private Number getMetric(String name, ApplicationId id) {
         return metrics.getMetric((dimensions) -> id.tenant().value().equals(dimensions.get("tenant")) &&
                                                  appDimension(id).equals(dimensions.get("app")),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
index 9677df6fd18..72b26aca588 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java
@@ -1,10 +1,9 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.zone.UpgradePolicy;
 import com.yahoo.config.provision.zone.ZoneApi;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -59,6 +58,7 @@ public class SystemUpgraderTest {
         systemUpgrader.maintain();
         assertCurrentVersion(SystemApplication.configServer, version1, zone1, zone2, zone3, zone4);
         assertCurrentVersion(SystemApplication.proxy, version1, zone1, zone2, zone3, zone4);
+        assertSystemVersion(version1);
 
         // Controller upgrades
         Version version2 = Version.fromString("6.6");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
index a1e22b4fc64..82837c20020 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
@@ -1,20 +1,23 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
+import java.nio.file.Files;
+import java.nio.file.Paths;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
-import java.util.stream.Collectors;
 
 import static java.time.temporal.ChronoUnit.MILLIS;
 import static org.junit.Assert.assertEquals;
@@ -36,9 +39,9 @@ public class VersionStatusSerializerTest {
                         ApplicationId.from("tenant2", "success2", "default"))
         );
         vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false,
-                                           true, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
+                                           true, nodeVersions(Version.fromString("5.0"), Instant.ofEpochMilli(123), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
         vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true,
-                                           false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
+                                           false, nodeVersions(Version.fromString("5.0"), Instant.ofEpochMilli(456), "cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
         VersionStatus status = new VersionStatus(vespaVersions);
         VersionStatusSerializer serializer = new VersionStatusSerializer();
         VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status));
@@ -53,14 +56,47 @@ public class VersionStatusSerializerTest {
             assertEquals(a.isSystemVersion(), b.isSystemVersion());
             assertEquals(a.isReleased(), b.isReleased());
             assertEquals(a.statistics(), b.statistics());
-            assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames());
+            assertEquals(a.nodeVersions(), b.nodeVersions());
             assertEquals(a.confidence(), b.confidence());
         }
 
     }
 
-    private static List<HostName> asHostnames(String... hostname) {
-        return Arrays.stream(hostname).map(HostName::from).collect(Collectors.toList());
+    @Test
+    public void testLegacySerialization() throws Exception {
+        var data = Files.readAllBytes(Paths.get("src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json"));
+        var serializer = new VersionStatusSerializer();
+        var deserializedStatus = serializer.fromSlime(SlimeUtils.jsonToSlime(data));
+
+        var statistics = new DeploymentStatistics(
+                Version.fromString("7.0"),
+                List.of(),
+                List.of(),
+                List.of()
+        );
+        var vespaVersion = new VespaVersion(statistics, "badc0ffee",
+                                            Instant.ofEpochMilli(123), true,
+                                            true, true,
+                                            nodeVersions(Version.emptyVersion, Instant.EPOCH, "cfg1", "cfg2", "cfg3"),
+                                            VespaVersion.Confidence.normal);
+
+        VespaVersion deserialized = deserializedStatus.versions().get(0);
+        assertEquals(vespaVersion.releaseCommit(), deserialized.releaseCommit());
+        assertEquals(vespaVersion.committedAt().truncatedTo(MILLIS), deserialized.committedAt());
+        assertEquals(vespaVersion.isControllerVersion(), deserialized.isControllerVersion());
+        assertEquals(vespaVersion.isSystemVersion(), deserialized.isSystemVersion());
+        assertEquals(vespaVersion.isReleased(), deserialized.isReleased());
+        assertEquals(vespaVersion.statistics(), deserialized.statistics());
+        assertEquals(vespaVersion.nodeVersions(), deserialized.nodeVersions());
+        assertEquals(vespaVersion.confidence(), deserialized.confidence());
+    }
+
+    private static NodeVersions nodeVersions(Version version, Instant changedAt, String... hostnames) {
+        var nodeVersions = NodeVersions.EMPTY;
+        for (var hostname : hostnames) {
+            nodeVersions = nodeVersions.with(new NodeVersion(HostName.from(hostname), version, changedAt));
+        }
+        return nodeVersions;
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json
new file mode 100644
index 00000000000..96ca22e1c1a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/testdata/version-status-legacy-format.json
@@ -0,0 +1,23 @@
+{
+  "versions": [
+    {
+      "releaseCommit": "badc0ffee",
+      "releasedAt": 123,
+      "isCurrentControllerVersion": true,
+      "isCurrentSystemVersion": true,
+      "isReleased": true,
+      "deploymentStatistics": {
+        "version": "7.0",
+        "failing": [],
+        "production": [],
+        "deploying": []
+      },
+      "confidence": "normal",
+      "configServerHostnames": [
+        "cfg1",
+        "cfg2",
+        "cfg3"
+      ]
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index 084b235943e..7140702ad60 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -1,26 +1,26 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.restapi.deployment;
 
-import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Test;
 
 import java.io.File;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.stream.Collectors;
 
 /**
  * @author bratseth
@@ -69,16 +69,15 @@ public class DeploymentApiTest extends ControllerContainerTest {
     private VersionStatus censorConfigServers(VersionStatus versionStatus, Controller controller) {
         List<VespaVersion> censored = new ArrayList<>();
         for (VespaVersion version : versionStatus.versions()) {
-            if (!version.systemApplicationHostnames().isEmpty()) {
+            if (version.nodeVersions().size() > 0) {
                 version = new VespaVersion(version.statistics(),
                                            version.releaseCommit(),
                                            version.committedAt(),
                                            version.isControllerVersion(),
                                            version.isSystemVersion(),
                                            version.isReleased(),
-                                           ImmutableSet.of("config1.test", "config2.test").stream()
-                                                       .map(HostName::from)
-                                                       .collect(Collectors.toSet()),
+                                           NodeVersions.EMPTY.with(new NodeVersion(HostName.from("config1.test"), version.versionNumber(), Instant.EPOCH))
+                                                             .with(new NodeVersion(HostName.from("config2.test"), version.versionNumber(), Instant.EPOCH)),
                                            VespaVersion.confidenceFrom(version.statistics(), controller)
                 );
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
index 17f90259fa8..01af1bd70dd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
@@ -6,92 +6,92 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         }
@@ -103,47 +103,47 @@
       "cloud": "cloud2",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-1-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-2-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-1-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-3-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-2-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
index 86bc272fcd1..dbaa6623fae 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
@@ -6,92 +6,92 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         }
@@ -103,47 +103,47 @@
       "cloud": "cloud2",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-1-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-2-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-1-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-3-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-2-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
index e8007fbf6c5..2b907c1156c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
@@ -6,47 +6,47 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-west-1",
           "environment": "prod",
           "region": "us-west-1"
         }
@@ -58,47 +58,47 @@
       "cloud": "cloud1",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-2-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-1-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-2-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-1-proxy-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-3-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-2-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-1-tenant-host-prod.us-east-3",
           "environment": "prod",
           "region": "us-east-3"
         }
@@ -110,47 +110,47 @@
       "cloud": "cloud2",
       "nodes": [
         {
-          "hostname": "node-1-configserver-host",
+          "hostname": "node-1-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-configserver-host",
+          "hostname": "node-2-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-configserver-host",
+          "hostname": "node-3-configserver-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-proxy-host",
+          "hostname": "node-1-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-proxy-host",
+          "hostname": "node-3-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-proxy-host",
+          "hostname": "node-2-proxy-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-2-tenant-host",
+          "hostname": "node-1-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-1-tenant-host",
+          "hostname": "node-3-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         },
         {
-          "hostname": "node-3-tenant-host",
+          "hostname": "node-2-tenant-host-prod.eu-west-1",
           "environment": "prod",
           "region": "eu-west-1"
         }