Measure platform and OS change duration per host

6 files changed, 121 insertions, 129 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
index 9c414ce8348..d70c65f343a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java
@@ -13,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
 import com.yahoo.vespa.hosted.controller.deployment.JobList;
 import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
+import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
 import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
@@ -20,6 +21,7 @@ import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.function.Function;
@@ -38,18 +40,21 @@ public class MetricsReporter extends Maintainer {
     public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades";
     public static final String DEPLOYMENT_BUILD_AGE_SECONDS = "deployment.buildAgeSeconds";
     public static final String DEPLOYMENT_WARNINGS = "deployment.warnings";
-    public static final String NODES_FAILING_SYSTEM_UPGRADE = "deployment.nodesFailingSystemUpgrade";
+    // TODO(mpolden): Remove these two metrics
+    public static final String NODES_FAILING_PLATFORM_UPGRADE = "deployment.nodesFailingSystemUpgrade";
     public static final String NODES_FAILING_OS_UPGRADE = "deployment.nodesFailingOsUpgrade";
+    public static final String OS_CHANGE_DURATION = "deployment.osChangeDuration";
+    public static final String PLATFORM_CHANGE_DURATION = "deployment.platformChangeDuration";
     public static final String REMAINING_ROTATIONS = "remaining_rotations";
     public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests";
 
-    // The time a node belonging to a system application can spend from being told to upgrade until the upgrade is
-    // completed. Nodes exceeding this time are counted as failures.
-    private static final Duration NODE_UPGRADE_TIMEOUT = Duration.ofMinutes(90);
+    // The time a system application node can spend after suspending for Vespa upgrade until the upgrade is completed.
+    // Nodes exceeding this budget are counted as failures.
+    private static final Duration PLATFORM_UPGRADE_BUDGET = Duration.ofMinutes(30);
 
-    // The time a single node can spend performing an OS upgrade after being told to upgrade. Nodes exceeding this time
-    // multiplied by the number of nodes upgrading are counted as failures.
-    private static final Duration OS_UPGRADE_TIME_ALLOWANCE_PER_NODE = Duration.ofMinutes(30);
+    // The time a system application node can spend after suspending foor OS upgrade until the upgrade is completed.
+    // Nodes exceeding this budget are counted as failures.
+    private static final Duration OS_UPGRADE_BUDGET = Duration.ofMinutes(30);
 
     private final Metric metric;
     private final Clock clock;
@@ -65,7 +70,7 @@ public class MetricsReporter extends Maintainer {
         reportDeploymentMetrics();
         reportRemainingRotations();
         reportQueuedNameServiceRequests();
-        reportNodesFailingUpgrade();
+        reportChangeDurations();
     }
 
     private void reportRemainingRotations() {
@@ -107,43 +112,44 @@ public class MetricsReporter extends Maintainer {
                    metric.createContext(Map.of()));
     }
 
-    private void reportNodesFailingUpgrade() {
-        metric.set(NODES_FAILING_SYSTEM_UPGRADE, nodesFailingSystemUpgrade(), metric.createContext(Map.of()));
-        metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade(), metric.createContext(Map.of()));
+    private void reportChangeDurations() {
+        var platformChangeDurations = platformChangeDurations();
+        var osChangeDurations = osChangeDurations();
+        var nodesFailingSystemUpgrade = platformChangeDurations.values().stream()
+                                                               .filter(duration -> duration.compareTo(PLATFORM_UPGRADE_BUDGET) > 0)
+                                                               .count();
+        var nodesFailingOsUpgrade = osChangeDurations.values().stream()
+                                                     .filter(duration -> duration.compareTo(OS_UPGRADE_BUDGET) > 0)
+                                                     .count();
+        metric.set(NODES_FAILING_PLATFORM_UPGRADE, nodesFailingSystemUpgrade, metric.createContext(Map.of()));
+        metric.set(NODES_FAILING_OS_UPGRADE, nodesFailingOsUpgrade, metric.createContext(Map.of()));
+        platformChangeDurations.forEach((nodeVersion, duration) -> {
+            metric.set(PLATFORM_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion)));
+        });
+        osChangeDurations.forEach((nodeVersion, duration) -> {
+            metric.set(OS_CHANGE_DURATION, duration.toSeconds(), metric.createContext(dimensions(nodeVersion)));
+        });
     }
 
-    private int nodesFailingSystemUpgrade() {
-        if (!controller().versionStatus().isUpgrading()) return 0;
-        return nodesFailingUpgrade(controller().versionStatus().versions(), (vespaVersion) -> {
-            if (vespaVersion.confidence() == VespaVersion.Confidence.broken) return NodeVersions.EMPTY;
-            return vespaVersion.nodeVersions();
-        }, NODE_UPGRADE_TIMEOUT);
+    private Map<NodeVersion, Duration> platformChangeDurations() {
+        return changeDurations(controller().versionStatus().versions(), VespaVersion::nodeVersions);
     }
 
-    private int nodesFailingOsUpgrade() {
-        var allNodeVersions = controller().osVersionStatus().versions().values();
-        var totalTimeout = 0L;
-        for (var nodeVersions : allNodeVersions) {
-            for (var nodeVersion : nodeVersions.asMap().values()) {
-                if (!nodeVersion.changing()) continue;
-                totalTimeout += OS_UPGRADE_TIME_ALLOWANCE_PER_NODE.toMillis();
-            }
-        }
-        return nodesFailingUpgrade(allNodeVersions, Function.identity(), Duration.ofMillis(totalTimeout));
+    private Map<NodeVersion, Duration> osChangeDurations() {
+        return changeDurations(controller().osVersionStatus().versions().values(), Function.identity());
     }
 
-    private <V> int nodesFailingUpgrade(Collection<V> collection, Function<V, NodeVersions> nodeVersionsFunction, Duration timeout) {
-        var nodesFailingUpgrade = 0;
-        var acceptableInstant = clock.instant().minus(timeout);
-        for (var object : collection) {
-            for (var nodeVersion : nodeVersionsFunction.apply(object).asMap().values()) {
-                if (!nodeVersion.changing()) continue;
-                if (nodeVersion.changedAt().isBefore(acceptableInstant)) nodesFailingUpgrade++;
+    private <V> Map<NodeVersion, Duration> changeDurations(Collection<V> versions, Function<V, NodeVersions> versionsGetter) {
+        var now = clock.instant();
+        var durations = new HashMap<NodeVersion, Duration>();
+        for (var version : versions) {
+            for (var nodeVersion : versionsGetter.apply(version).asMap().values()) {
+                durations.put(nodeVersion, nodeVersion.changeDuration(now));
             }
         }
-        return nodesFailingUpgrade;
+        return durations;
     }
-    
+
     private static double deploymentFailRatio(DeploymentStatusList statuses) {
         return statuses.asList().stream()
                        .mapToInt(status -> status.hasFailures() ? 1 : 0)
@@ -198,6 +204,11 @@ public class MetricsReporter extends Maintainer {
                       "app",application.application().value() + "." + application.instance().value());
     }
 
+    private static Map<String, String> dimensions(NodeVersion nodeVersion) {
+        return Map.of("host", nodeVersion.hostname().value(),
+                      "zone", nodeVersion.zone().value());
+    }
+
 }
 
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java
index 6e9def40e44..f9f8de96591 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NodeVersionSerializer.java
@@ -11,8 +11,6 @@ import com.yahoo.slime.Inspector;
 import com.yahoo.vespa.hosted.controller.versions.NodeVersion;
 import com.yahoo.vespa.hosted.controller.versions.NodeVersions;
 
-import java.time.Instant;
-
 /**
  * Serializer for {@link com.yahoo.vespa.hosted.controller.versions.NodeVersion}.
  *
@@ -30,7 +28,7 @@ public class NodeVersionSerializer {
     private static final String hostnameField = "hostname";
     private static final String zoneField = "zone";
     private static final String wantedVersionField = "wantedVersion";
-    private static final String changedAtField = "changedAt";
+    private static final String suspendedAtField = "suspendedAt";
 
     public void nodeVersionsToSlime(NodeVersions nodeVersions, Cursor array) {
         for (var nodeVersion : nodeVersions.asMap().values()) {
@@ -38,7 +36,8 @@ public class NodeVersionSerializer {
             nodeVersionObject.setString(hostnameField, nodeVersion.hostname().value());
             nodeVersionObject.setString(zoneField, nodeVersion.zone().value());
             nodeVersionObject.setString(wantedVersionField, nodeVersion.wantedVersion().toFullString());
-            nodeVersionObject.setLong(changedAtField, nodeVersion.changedAt().toEpochMilli());
+            nodeVersion.suspendedAt().ifPresent(suspendedAt -> nodeVersionObject.setLong(suspendedAtField,
+                                                                                         suspendedAt.toEpochMilli()));
         }
     }
 
@@ -48,8 +47,8 @@ public class NodeVersionSerializer {
             var hostname = HostName.from(entry.field(hostnameField).asString());
             var zone = ZoneId.from(entry.field(zoneField).asString());
             var wantedVersion = Version.fromString(entry.field(wantedVersionField).asString());
-            var changedAt = Instant.ofEpochMilli(entry.field(changedAtField).asLong());
-            nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, changedAt));
+            var suspendedAt = Serializers.optionalInstant(entry.field(suspendedAtField));
+            nodeVersions.put(hostname, new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt));
         });
         return new NodeVersions(nodeVersions.build());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
index d82cddb4779..4a00add411f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersion.java
@@ -5,8 +5,10 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.Objects;
+import java.util.Optional;
 
 /**
  * Version information for a node allocated to a {@link com.yahoo.vespa.hosted.controller.application.SystemApplication}.
@@ -21,14 +23,15 @@ public class NodeVersion {
     private final ZoneId zone;
     private final Version currentVersion;
     private final Version wantedVersion;
-    private final Instant changedAt;
+    private final Optional<Instant> suspendedAt;
 
-    public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion, Instant changedAt) {
+    public NodeVersion(HostName hostname, ZoneId zone, Version currentVersion, Version wantedVersion,
+                       Optional<Instant> suspendedAt) {
         this.hostname = Objects.requireNonNull(hostname, "hostname must be non-null");
         this.zone = Objects.requireNonNull(zone, "zone must be non-null");
         this.currentVersion = Objects.requireNonNull(currentVersion, "version must be non-null");
         this.wantedVersion = Objects.requireNonNull(wantedVersion, "wantedVersion must be non-null");
-        this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null");
+        this.suspendedAt = Objects.requireNonNull(suspendedAt, "suspendedAt must be non-null");
     }
 
     /** Hostname of this */
@@ -51,31 +54,39 @@ public class NodeVersion {
         return wantedVersion;
     }
 
-    /** Returns whether this is changing (upgrading or downgrading) */
-    public boolean changing() {
-        return !currentVersion.equals(wantedVersion);
+    /** Returns the duration of the change in this, measured relative to instant */
+    public Duration changeDuration(Instant instant) {
+        if (!changing()) return Duration.ZERO;
+        if (suspendedAt.isEmpty()) return Duration.ZERO; // Node hasn't suspended to apply the change yet
+        return Duration.between(suspendedAt.get(), instant).abs();
     }
 
-    /** The most recent time the version of this changed */
-    public Instant changedAt() {
-        return changedAt;
+    /** The most recent time the node referenced by this suspended. This is empty if the node is not suspended. */
+    public Optional<Instant> suspendedAt() {
+        return suspendedAt;
     }
 
     /** Returns a copy of this with current version set to given version */
-    public NodeVersion withCurrentVersion(Version version, Instant changedAt) {
+    public NodeVersion withCurrentVersion(Version version) {
         if (currentVersion.equals(version)) return this;
-        return new NodeVersion(hostname, zone, version, wantedVersion, changedAt);
+        return new NodeVersion(hostname, zone, version, wantedVersion, suspendedAt);
     }
 
     /** Returns a copy of this with wanted version set to given version */
     public NodeVersion withWantedVersion(Version version) {
         if (wantedVersion.equals(version)) return this;
-        return new NodeVersion(hostname, zone, currentVersion, version, changedAt);
+        return new NodeVersion(hostname, zone, currentVersion, version, suspendedAt);
+    }
+
+    /** Returns a copy of this with wanted version set to given version */
+    public NodeVersion withSuspendedAt(Optional<Instant> suspendedAt) {
+        if (suspendedAt.equals(this.suspendedAt)) return this;
+        return new NodeVersion(hostname, zone, currentVersion, wantedVersion, suspendedAt);
     }
 
     @Override
     public String toString() {
-        return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", changedAt=" + changedAt + "]";
+        return hostname + ": " + currentVersion + " -> " + wantedVersion + " [zone=" + zone + ", suspendedAt=" + suspendedAt.map(Instant::toString).orElse("<not suspended>") + "]";
     }
 
     @Override
@@ -87,12 +98,17 @@ public class NodeVersion {
                zone.equals(that.zone) &&
                currentVersion.equals(that.currentVersion) &&
                wantedVersion.equals(that.wantedVersion) &&
-               changedAt.equals(that.changedAt);
+               suspendedAt.equals(that.suspendedAt);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(hostname, zone, currentVersion, wantedVersion, changedAt);
+        return Objects.hash(hostname, zone, currentVersion, wantedVersion, suspendedAt);
+    }
+
+    /** Returns whether this is changing (upgrading or downgrading) */
+    private boolean changing() {
+        return !currentVersion.equals(wantedVersion);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
index 3ab96e03bcd..4ce0a35e96f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/NodeVersions.java
@@ -7,6 +7,7 @@ import com.google.common.collect.ListMultimap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
 
+import java.util.Collection;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -20,12 +21,10 @@ import java.util.function.Predicate;
  */
 public class NodeVersions {
 
-    public static final NodeVersions EMPTY = new NodeVersions(ImmutableMap.of());
-
     private final ImmutableMap<HostName, NodeVersion> nodeVersions;
 
-    public NodeVersions(ImmutableMap<HostName, NodeVersion> nodeVersions) {
-        this.nodeVersions = Objects.requireNonNull(nodeVersions);
+    public NodeVersions(Map<HostName, NodeVersion> nodeVersions) {
+        this.nodeVersions = ImmutableMap.copyOf(Objects.requireNonNull(nodeVersions));
     }
 
     public Map<HostName, NodeVersion> asMap() {
@@ -48,7 +47,7 @@ public class NodeVersions {
 
     /** Returns a copy of this containing only node versions of given version */
     public NodeVersions matching(Version version) {
-        return filter(nodeVersion -> nodeVersion.currentVersion().equals(version));
+        return copyOf(nodeVersions.values(), nodeVersion -> nodeVersion.currentVersion().equals(version));
     }
 
     /** Returns number of node versions in this */
@@ -56,31 +55,6 @@ public class NodeVersions {
         return nodeVersions.size();
     }
 
-    /** Returns a copy of this containing only the given node versions */
-    public NodeVersions with(List<NodeVersion> nodeVersions) {
-        var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder();
-        for (var nodeVersion : nodeVersions) {
-            var existing = this.nodeVersions.get(nodeVersion.hostname());
-            if (existing != null) {
-                newNodeVersions.put(nodeVersion.hostname(), existing.withCurrentVersion(nodeVersion.currentVersion(),
-                                                                                        nodeVersion.changedAt())
-                                                                    .withWantedVersion(nodeVersion.wantedVersion()));
-            } else {
-                newNodeVersions.put(nodeVersion.hostname(), nodeVersion);
-            }
-        }
-        return new NodeVersions(newNodeVersions.build());
-    }
-
-    private NodeVersions filter(Predicate<NodeVersion> predicate) {
-        var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder();
-        for (var kv : nodeVersions.entrySet()) {
-            if (!predicate.test(kv.getValue())) continue;
-            newNodeVersions.put(kv.getKey(), kv.getValue());
-        }
-        return new NodeVersions(newNodeVersions.build());
-    }
-
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
@@ -94,4 +68,21 @@ public class NodeVersions {
         return Objects.hash(nodeVersions);
     }
 
+    public static NodeVersions copyOf(List<NodeVersion> nodeVersions) {
+        return copyOf(nodeVersions, (ignored) -> true);
+    }
+
+    public static NodeVersions copyOf(Map<HostName, NodeVersion> nodeVersions) {
+        return new NodeVersions(nodeVersions);
+    }
+
+    private static NodeVersions copyOf(Collection<NodeVersion> nodeVersions, Predicate<NodeVersion> predicate) {
+        var newNodeVersions = ImmutableMap.<HostName, NodeVersion>builder();
+        for (var nodeVersion : nodeVersions) {
+            if (!predicate.test(nodeVersion)) continue;
+            newNodeVersions.put(nodeVersion.hostname(), nodeVersion);
+        }
+        return new NodeVersions(newNodeVersions.build());
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
index 1773a9c122e..1dffd1383bd 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java
@@ -16,7 +16,6 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
@@ -59,9 +58,7 @@ public class OsVersionStatus {
 
     /** Compute the current OS versions in this system. This is expensive and should be called infrequently */
     public static OsVersionStatus compute(Controller controller) {
-        var osVersionStatus = controller.osVersionStatus();
         var osVersions = new HashMap<OsVersion, List<NodeVersion>>();
-        var now = controller.clock().instant();
         controller.osVersions().forEach(osVersion -> osVersions.put(osVersion, new ArrayList<>()));
 
         for (var application : SystemApplication.all()) {
@@ -71,19 +68,16 @@ public class OsVersionStatus {
                                                 .targetVersionsOf(zone.getId())
                                                 .osVersion(application.nodeType())
                                                 .orElse(Version.emptyVersion);
-                controller.serviceRegistry().configServer().nodeRepository()
-                          .list(zone.getId(), application.id()).stream()
-                          .filter(node -> OsUpgrader.eligibleForUpgrade(node, application))
-                          .map(node -> new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(), targetOsVersion, now))
-                          .forEach(nodeVersion -> {
-                              var newNodeVersion = osVersionStatus.of(nodeVersion.hostname())
-                                                                  .map(nv -> nv.withCurrentVersion(nodeVersion.currentVersion(), now)
-                                                                               .withWantedVersion(nodeVersion.wantedVersion()))
-                                                                  .orElse(nodeVersion);
-                              var version = new OsVersion(newNodeVersion.currentVersion(), zone.getCloudName());
-                              osVersions.putIfAbsent(version, new ArrayList<>());
-                              osVersions.get(version).add(newNodeVersion);
-                          });
+
+                for (var node : controller.serviceRegistry().configServer().nodeRepository().list(zone.getId(), application.id())) {
+                    if (!OsUpgrader.eligibleForUpgrade(node, application)) continue;
+                    var suspendedAt = node.suspendedSince();
+                    var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), node.currentOsVersion(),
+                                                      targetOsVersion, suspendedAt);
+                    var osVersion = new OsVersion(nodeVersion.currentVersion(), zone.getCloudName());
+                    osVersions.putIfAbsent(osVersion, new ArrayList<>());
+                    osVersions.get(osVersion).add(nodeVersion);
+                }
             }
         }
 
@@ -98,15 +92,6 @@ public class OsVersionStatus {
         return new OsVersionStatus(newOsVersions.build());
     }
 
-    /** Returns version of node identified by given host name */
-    private Optional<NodeVersion> of(HostName hostname) {
-        return versions.values().stream()
-                       .map(nodeVersions -> nodeVersions.asMap().get(hostname))
-                       .map(Optional::ofNullable)
-                       .flatMap(Optional::stream)
-                       .findFirst();
-    }
-
     private static List<ZoneApi> zonesToUpgrade(Controller controller) {
         return controller.zoneRegistry().osUpgradePolicies().stream()
                          .flatMap(upgradePolicy -> upgradePolicy.asList().stream())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 0868d7ca695..7618e8210e3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -8,23 +8,15 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
-import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
-import com.yahoo.vespa.hosted.controller.deployment.JobStatus;
-import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
 import com.yahoo.vespa.hosted.controller.maintenance.SystemUpgrader;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.HashMap;
+import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.logging.Level;
@@ -149,10 +141,7 @@ public class VersionStatus {
     }
 
     private static NodeVersions findSystemApplicationVersions(Controller controller) {
-        var nodeVersions = controller.versionStatus().systemVersion()
-                                     .map(VespaVersion::nodeVersions)
-                                     .orElse(NodeVersions.EMPTY);
-        var newNodeVersions = new ArrayList<NodeVersion>();
+        var nodeVersions = new LinkedHashMap<HostName, NodeVersion>();
         for (var zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) {
             for (var application : SystemApplication.all()) {
                 var nodes = controller.serviceRegistry().configServer().nodeRepository()
@@ -165,15 +154,16 @@ public class VersionStatus {
                     log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone.getId() +
                                               " has not converged");
                 }
-                var now = controller.clock().instant();
                 for (var node : nodes) {
                     // Only use current node version if config has converged
-                    Version version = configConverged ? node.currentVersion() : controller.systemVersion();
-                    newNodeVersions.add(new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(), now));
+                    var version = configConverged ? node.currentVersion() : controller.systemVersion();
+                    var nodeVersion = new NodeVersion(node.hostname(), zone.getId(), version, node.wantedVersion(),
+                                                      node.suspendedSince());
+                    nodeVersions.put(nodeVersion.hostname(), nodeVersion);
                 }
             }
         }
-        return nodeVersions.with(newNodeVersions);
+        return NodeVersions.copyOf(nodeVersions);
     }
 
     private static ListMultimap<ControllerVersion, HostName> findControllerVersions(Controller controller) {