Merge pull request #3746 from vespa-engine/mpolden/freeze-confidence

Freeze confidence for previous versions

14 files changed, 361 insertions, 161 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 4f5c2ead4da..35551b50953 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode;
 import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
+import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
@@ -40,7 +41,6 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
-import java.util.concurrent.atomic.AtomicReference;
 import java.util.logging.Logger;
 
 /**
@@ -64,15 +64,7 @@ public class Controller extends AbstractComponent {
     private final CuratorDb curator;
     private final ApplicationController applicationController;
     private final TenantController tenantController;
-    
-    /** 
-     * Status of Vespa versions across the system. 
-     * This is expensive to maintain so that is done periodically by a maintenance job 
-     */
-    private final AtomicReference<VersionStatus> versionStatus;
-    
     private final Clock clock;
-
     private final RotationRepository rotationRepository;
     private final GitHub gitHub;
     private final EntityService entityService;
@@ -141,7 +133,6 @@ public class Controller extends AbstractComponent {
         applicationController = new ApplicationController(this, db, curator, rotationRepository, athens.zmsClientFactory(),
                                                           nameService, configServerClient, routingGenerator, clock);
         tenantController = new TenantController(this, db, curator, entityService);
-        versionStatus = new AtomicReference<>(VersionStatus.empty());
     }
     
     /** Returns the instance controlling tenants */
@@ -191,11 +182,7 @@ public class Controller extends AbstractComponent {
                              "sort:!('@timestamp',desc))";
 
         URI kibanaPath = URI.create(kibanaQuery);
-        if (kibanaHost.isPresent()) {
-            return kibanaHost.get().resolve(kibanaPath);
-        } else {
-            return null;
-        }
+        return kibanaHost.map(uri -> uri.resolve(kibanaPath)).orElse(null);
     }
 
     public Set<URI> getRotationUris(ApplicationId id) {
@@ -235,17 +222,19 @@ public class Controller extends AbstractComponent {
             ! newStatus.systemVersion().equals(currentStatus.systemVersion())) {
             log.info("Changing system version from " + printableVersion(currentStatus.systemVersion()) +
                      " to " + printableVersion(newStatus.systemVersion()));
-            curator.writeSystemVersion(newStatus.systemVersion().get().versionNumber());
         }
-
-        this.versionStatus.set(newStatus); 
+        curator.writeVersionStatus(newStatus);
     }
     
     /** Returns the latest known version status. Calling this is free but the status may be slightly out of date. */
-    public VersionStatus versionStatus() { return versionStatus.get(); }
+    public VersionStatus versionStatus() { return curator.readVersionStatus(); }
     
     /** Returns the current system version: The controller should drive towards running all applications on this version */
-    public Version systemVersion() { return curator.readSystemVersion(); }
+    public Version systemVersion() {
+        return versionStatus().systemVersion()
+                .map(VespaVersion::versionNumber)
+                .orElse(Vtag.currentVersion);
+    }
 
     public MetricsService metricsService() { return metricsService; }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index a70e31d9de8..2ca662ccb66 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -4,20 +4,21 @@ package com.yahoo.vespa.hosted.controller.persistence;
 import com.google.inject.Inject;
 import com.yahoo.cloud.config.ClusterInfoConfig;
 import com.yahoo.cloud.config.ZookeeperServerConfig;
-import com.yahoo.component.Version;
-import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.net.HostName;
 import com.yahoo.path.Path;
 import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.zookeeper.ZooKeeperServer;
 
+import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.nio.ByteBuffer;
-import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayDeque;
 import java.util.Collections;
@@ -148,18 +149,6 @@ public class CuratorDb {
 
     // -------------- Read and write --------------------------------------------------
 
-    public Version readSystemVersion() {
-        Optional<byte[]> data = curator.getData(systemVersionPath());
-        if (! data.isPresent() || data.get().length == 0) return Vtag.currentVersion;
-        return Version.fromString(new String(data.get(), StandardCharsets.UTF_8));
-    }
-
-    public void writeSystemVersion(Version version) {
-        NestedTransaction transaction = new NestedTransaction();
-        curator.set(systemVersionPath(), version.toString().getBytes(StandardCharsets.UTF_8));
-        transaction.commit();
-    }
-
     public Set<String> readInactiveJobs() {
         try {
             Optional<byte[]> data = curator.getData(inactiveJobsPath());
@@ -199,7 +188,7 @@ public class CuratorDb {
     }
 
     public double readUpgradesPerMinute() {
-        Optional<byte[]> n = curator.getData(upgradePerMinutePath());
+        Optional<byte[]> n = curator.getData(upgradesPerMinutePath());
         if (!n.isPresent() || n.get().length == 0) {
             return 0.5; // Default if value has never been written
         }
@@ -211,10 +200,34 @@ public class CuratorDb {
             throw new IllegalArgumentException("Upgrades per minute must be >= 0");
         }
         NestedTransaction transaction = new NestedTransaction();
-        curator.set(upgradePerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array());
+        curator.set(upgradesPerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array());
+        transaction.commit();
+    }
+
+    public void writeVersionStatus(VersionStatus status) {
+        VersionStatusSerializer serializer = new VersionStatusSerializer();
+        NestedTransaction transaction = new NestedTransaction();
+        try {
+            // TODO: Removes unused data. Remove after October 2017
+            if (curator.getData(systemVersionPath()).isPresent()) {
+                curator.delete(systemVersionPath());
+            }
+            curator.set(versionStatusPath(), SlimeUtils.toJsonBytes(serializer.toSlime(status)));
+        } catch (IOException e) {
+            throw new UncheckedIOException("Failed to serialize version status", e);
+        }
         transaction.commit();
     }
 
+    public VersionStatus readVersionStatus() {
+        Optional<byte[]> data = curator.getData(versionStatusPath());
+        if (!data.isPresent() || data.get().length == 0) {
+            return VersionStatus.empty(); // Default if status has never been written
+        }
+        VersionStatusSerializer serializer = new VersionStatusSerializer();
+        return serializer.fromSlime(SlimeUtils.jsonToSlime(data.get()));
+    }
+
     public Optional<byte[]> readProvisionState(String provisionId) {
         return curator.getData(provisionStatePath().append(provisionId));
     }
@@ -264,10 +277,12 @@ public class CuratorDb {
         return root.append("jobQueues").append(jobType.name());
     }
 
-    private Path upgradePerMinutePath() {
+    private Path upgradesPerMinutePath() {
         return root.append("upgrader").append("upgradesPerMinute");
     }
 
+    private Path versionStatusPath() { return root.append("versionStatus"); }
+
     private Path provisionStatePath() {
         return root.append("provisioning").append("states");
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
new file mode 100644
index 00000000000..e0bc592c82c
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java
@@ -0,0 +1,118 @@
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Serializes VersionStatus to and from slime
+ *
+ * @author mpolden
+ */
+public class VersionStatusSerializer {
+
+    // VersionStatus fields
+    private static final String versionsField = "versions";
+
+    // VespaVersion fields
+    private static final String releaseCommitField = "releaseCommit";
+    private static final String releasedAtField = "releasedAt";
+    private static final String isCurrentSystemVersionField = "isCurrentSystemVersion";
+    private static final String deploymentStatisticsField = "deploymentStatistics";
+    private static final String confidenceField = "confidence";
+    private static final String configServersField = "configServerHostnames";
+
+    // DeploymentStatistics fields
+    private static final String versionField = "version";
+    private static final String failingField = "failing";
+    private static final String productionField = "production";
+
+    public Slime toSlime(VersionStatus status) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        versionsToSlime(status.versions(), root.setArray(versionsField));
+        return slime;
+    }
+
+    public VersionStatus fromSlime(Slime slime) {
+        Inspector root = slime.get();
+        return new VersionStatus(vespaVersionsFromSlime(root.field(versionsField)));
+    }
+
+    private void versionsToSlime(List<VespaVersion> versions, Cursor array) {
+        versions.forEach(version -> vespaVersionToSlime(version, array.addObject()));
+    }
+
+    private void vespaVersionToSlime(VespaVersion version, Cursor object) {
+        object.setString(releaseCommitField, version.releaseCommit());
+        object.setLong(releasedAtField, version.releasedAt().toEpochMilli());
+        object.setBool(isCurrentSystemVersionField, version.isCurrentSystemVersion());
+        deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField));
+        object.setString(confidenceField, version.confidence().name());
+        configServersToSlime(version.configServerHostnames(), object.setArray(configServersField));
+    }
+
+    private void configServersToSlime(Set<String> configServerHostnames, Cursor array) {
+        configServerHostnames.forEach(array::addString);
+    }
+
+    private void deploymentStatisticsToSlime(DeploymentStatistics statistics, Cursor object) {
+        object.setString(versionField, statistics.version().toString());
+        applicationsToSlime(statistics.failing(), object.setArray(failingField));
+        applicationsToSlime(statistics.production(), object.setArray(productionField));
+    }
+
+    private void applicationsToSlime(List<ApplicationId> applications, Cursor array) {
+        applications.forEach(application -> array.addString(application.serializedForm()));
+    }
+
+    private List<VespaVersion> vespaVersionsFromSlime(Inspector array) {
+        List<VespaVersion> versions = new ArrayList<>();
+        array.traverse((ArrayTraverser) (i, object) -> versions.add(vespaVersionFromSlime(object)));
+        return Collections.unmodifiableList(versions);
+    }
+
+    private VespaVersion vespaVersionFromSlime(Inspector object) {
+        return new VespaVersion(deploymentStatisticsFromSlime(object.field(deploymentStatisticsField)),
+                                object.field(releaseCommitField).asString(),
+                                Instant.ofEpochMilli(object.field(releasedAtField).asLong()),
+                                object.field(isCurrentSystemVersionField).asBool(),
+                                configServersFromSlime(object.field(configServersField)),
+                                VespaVersion.Confidence.valueOf(object.field(confidenceField).asString())
+        );
+    }
+
+    private Set<String> configServersFromSlime(Inspector array) {
+        Set<String> configServerHostnames = new LinkedHashSet<>();
+        array.traverse((ArrayTraverser) (i, entry) -> configServerHostnames.add(entry.asString()));
+        return Collections.unmodifiableSet(configServerHostnames);
+    }
+
+    private DeploymentStatistics deploymentStatisticsFromSlime(Inspector object) {
+        return new DeploymentStatistics(Version.fromString(object.field(versionField).asString()),
+                                        applicationsFromSlime(object.field(failingField)),
+                                        applicationsFromSlime(object.field(productionField)));
+    }
+
+    private List<ApplicationId> applicationsFromSlime(Inspector array) {
+        List<ApplicationId> applications = new ArrayList<>();
+        array.traverse((ArrayTraverser) (i, entry) -> applications.add(
+                ApplicationId.fromSerializedForm(entry.asString()))
+        );
+        return Collections.unmodifiableList(applications);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
index fbd1a74c12c..6174a017a54 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/DeploymentStatistics.java
@@ -6,6 +6,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 
 import java.util.List;
+import java.util.Objects;
 
 /**
  * Statistics about deployments on a platform version. This is immutable.
@@ -18,8 +19,9 @@ public class DeploymentStatistics {
     private final ImmutableList<ApplicationId> failing;
     private final ImmutableList<ApplicationId> production;
 
-    private DeploymentStatistics(Version version,
-                                 List<ApplicationId> failingApplications, List<ApplicationId> production) {
+    /** DO NOT USE. Public for serialization purposes */
+    public DeploymentStatistics(Version version, List<ApplicationId> failingApplications,
+                                List<ApplicationId> production) {
         this.version = version;
         this.failing = ImmutableList.copyOf(failingApplications);
         this.production = ImmutableList.copyOf(production);
@@ -59,4 +61,18 @@ public class DeploymentStatistics {
         return b.build();
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof DeploymentStatistics)) return false;
+        DeploymentStatistics that = (DeploymentStatistics) o;
+        return Objects.equals(version, that.version) &&
+               Objects.equals(failing, that.failing) &&
+               Objects.equals(production, that.production);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(version, failing, production);
+    }
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index c3f3773cde6..c3b2da42861 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -5,9 +5,9 @@ import com.google.common.collect.ImmutableList;
 import com.yahoo.collections.ListMap;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
-import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
@@ -46,7 +46,7 @@ public class VersionStatus {
 
     private final ImmutableList<VespaVersion> versions;
     
-    /** Create a version status. DO NOT USE: Public for testing only */
+    /** Create a version status. DO NOT USE: Public for testing and serialization only */
     public VersionStatus(List<VespaVersion> versions) {
         this.versions = ImmutableList.copyOf(versions);
     }
@@ -175,11 +175,30 @@ public class VersionStatus {
                                               Collection<String> configServerHostnames,
                                               Controller controller) {
         GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString());
+        Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime());
+        VespaVersion.Confidence confidence;
+        // Always compute confidence for system version
+        if (isSystemVersion) {
+            confidence = VespaVersion.confidenceFrom(statistics, controller, releasedAt);
+        } else {
+            // Keep existing confidence for non-system versions if already computed
+            confidence = confidenceFor(statistics.version(), controller)
+                    .orElse(VespaVersion.confidenceFrom(statistics, controller, releasedAt));
+        }
         return new VespaVersion(statistics,
-                                gitSha.sha, Instant.ofEpochMilli(gitSha.commit.author.date.getTime()),
+                                gitSha.sha, releasedAt,
                                 isSystemVersion,
                                 configServerHostnames,
-                                controller);
+                                confidence
+        );
+    }
+
+    /** Returns the current confidence for the given version */
+    private static Optional<VespaVersion.Confidence> confidenceFor(Version version, Controller controller) {
+        return controller.versionStatus().versions().stream()
+                .filter(v -> version.equals(v.versionNumber()))
+                .map(VespaVersion::confidence)
+                .findFirst();
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index 3208f4d09c6..c1b9c045fbe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.versions;
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
-import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 
@@ -12,6 +11,8 @@ import java.time.Instant;
 import java.util.Collection;
 import java.util.Set;
 
+import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
+
 /**
  * Information about a particular Vespa version.
  * VespaVersions are identified by their version number and ordered by increasing version numbers.
@@ -26,23 +27,22 @@ public class VespaVersion implements Comparable<VespaVersion> {
     private final Instant releasedAt;
     private final boolean isCurrentSystemVersion;
     private final DeploymentStatistics statistics;
-    private final Confidence confidence;
     private final ImmutableSet<String> configServerHostnames;
+    private final Confidence confidence;
 
-    public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt, 
+    public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant releasedAt,
                         boolean isCurrentSystemVersion, Collection<String> configServerHostnames,
-                        Controller controller) {
+                        Confidence confidence) {
         this.statistics = statistics;
         this.releaseCommit = releaseCommit;
         this.releasedAt = releasedAt;
         this.isCurrentSystemVersion = isCurrentSystemVersion;
         this.configServerHostnames = ImmutableSet.copyOf(configServerHostnames);
-        this.confidence = deduceConfidenceFrom(statistics, controller, releasedAt);
+        this.confidence = confidence;
     }
 
-    private static Confidence deduceConfidenceFrom(DeploymentStatistics statistics, 
-                                                   Controller controller, 
-                                                   Instant releasedAt) {
+    public static Confidence confidenceFrom(DeploymentStatistics statistics, Controller controller,
+                                            Instant releasedAt) {
         // 'production on this': All deployment jobs upgrading to this version have completed without failure
         ApplicationList productionOnThis = ApplicationList.from(statistics.production(), controller.applications())
                                                           .notUpgradingTo(statistics.version())
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
index 9db852374b8..9228e83bbc6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
@@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.vespa.serviceview.bindings.ClusterView;
@@ -28,7 +28,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -40,15 +39,11 @@ import java.util.UUID;
  */
 public class ConfigServerClientMock extends AbstractComponent implements ConfigServerClient {
 
-    private Map<ApplicationId, byte[]> applicationContent = new HashMap<>();
-    private Map<ApplicationId, String> applicationInstances = new HashMap<>();
-    private Map<ApplicationId, Boolean> applicationActivated = new HashMap<>();
-    private Set<ApplicationId> applicationRestarted = new HashSet<>();
-    private Set<String> hostsExplicitlyRestarted = new HashSet<>();
-    private Map<String, EndpointStatus> endpoints = new HashMap<>();
-
-    private Map<URI, Version> configServerVersions = new HashMap<>();
-    private Version defaultConfigServerVersion = new Version(6, 1, 0);
+    private final Map<ApplicationId, String> applicationInstances = new HashMap<>();
+    private final Map<ApplicationId, Boolean> applicationActivated = new HashMap<>();
+    private final Map<String, EndpointStatus> endpoints = new HashMap<>();
+    private final Map<URI, Version> versions = new HashMap<>();
+    private Version defaultVersion = new Version(6, 1, 0);
 
     /** The exception to throw on the next prepare run, or null to continue normally */
     private RuntimeException prepareException = null;
@@ -64,18 +59,27 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
     public Map<ApplicationId, Boolean> activated() {
         return Collections.unmodifiableMap(applicationActivated);
     }
+
+    public void throwOnNextPrepare(RuntimeException prepareException) {
+        this.prepareException = prepareException;
+    }
+
+    /**
+     * Returns the (initially empty) mutable map of config server urls to versions.
+     * This API will return defaultVersion as response to any version(url) call for versions not added to the map.
+     */
+    public Map<URI, Version> versions() {
+        return versions;
+    }
     
     @Override
     public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) {
         lastPrepareVersion = deployOptions.vespaVersion.map(Version::new);
-        
         if (prepareException != null) {
             RuntimeException prepareException = this.prepareException;
             this.prepareException = null;
             throw prepareException;
         }
-
-        applicationContent.put(deployment.applicationId(), content);
         applicationActivated.put(deployment.applicationId(), false);
         applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080");
 
@@ -111,20 +115,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
         };
     }
 
-    public void throwOnNextPrepare(RuntimeException prepareException) {
-        this.prepareException = prepareException;
-    }
-
-    /**
-     * Returns the (initially empty) mutable map of config server urls to versions.
-     * This API will return defaultConfigserverVersion as response to any version(url) call for versions not added to the map.
-     */
-    public Map<URI, Version> configServerVersions() {
-        return configServerVersions;
-    }
-    
-    public Version getDefaultConfigServerVersion() { return defaultConfigServerVersion; }
-    public void setDefaultConfigServerVersion(Version version) { defaultConfigServerVersion = version; }
+    /** Set the default config server version */
+    public void setDefaultVersion(Version version) { this.defaultVersion = version; }
     
     @Override
     public List<String> getNodeQueryHost(DeploymentId deployment, String type) {
@@ -137,16 +129,11 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
 
     @Override
     public void restart(DeploymentId deployment, Optional<Hostname> hostname) {
-        applicationRestarted.add(deployment.applicationId());
-        if (hostname.isPresent()) {
-            hostsExplicitlyRestarted.add(hostname.get().id());
-        }
     }
 
     @Override
     public void deactivate(DeploymentId deployment) {
         applicationActivated.remove(deployment.applicationId());
-        applicationContent.remove(deployment.applicationId());
         applicationInstances.remove(deployment.applicationId());
     }
 
@@ -197,7 +184,7 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
     
     @Override
     public Version version(URI configServerURI) {
-        return configServerVersions.getOrDefault(configServerURI, defaultConfigServerVersion);
+        return versions.getOrDefault(configServerURI, defaultVersion);
     }
 
     @Override
@@ -207,10 +194,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
 
     @Override
     public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) {
-        EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757l);
-        return endpoints.containsKey(endpoint)
-                ? endpoints.get(endpoint)
-                : result;
+        EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757L);
+        return endpoints.getOrDefault(endpoint, result);
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 6fc787d940e..b193fd7ef41 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -260,16 +260,19 @@ public class ControllerTest {
         Version systemVersion = controller.versionStatus().systemVersion().get().versionNumber();
         Version newSystemVersion = new Version(systemVersion.getMajor(), systemVersion.getMinor()+1, 0);
         VespaVersion newSystemVespaVersion = new VespaVersion(DeploymentStatistics.empty(newSystemVersion),
-                                                              "commit1", 
+                                                              "commit1",
                                                               Instant.now(),
                                                               true,
                                                               Collections.emptyList(),
-                                                              controller);
+                                                              VespaVersion.Confidence.low
+        );
         List<VespaVersion> versions = new ArrayList<>(controller.versionStatus().versions());
         for (int i = 0; i < versions.size(); i++) {
             VespaVersion c = versions.get(i);
             if (c.isCurrentSystemVersion())
-                versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(), false, c.configServerHostnames(), controller));
+                versions.set(i, new VespaVersion(c.statistics(), c.releaseCommit(), c.releasedAt(),
+                                                 false, c.configServerHostnames(),
+                                                 c.confidence()));
         }
         versions.add(newSystemVespaVersion);
         controller.updateVersionStatus(new VersionStatus(versions));
@@ -542,7 +545,7 @@ public class ControllerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.141.117");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
@@ -572,7 +575,7 @@ public class ControllerTest {
 
         // New version is released
         version = Version.fromString("6.142.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index be14947de2b..e45166b1c16 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -90,12 +90,17 @@ public class DeploymentTester {
                 .filter(c -> c instanceof Change.VersionChange)
                 .map(Change.VersionChange.class::cast);
     }
+
+    public void updateVersionStatus() {
+        controller().updateVersionStatus(VersionStatus.compute(controller(), tester.controller().systemVersion()));
+    }
     
     public void updateVersionStatus(Version currentVersion) {
         controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion));
     }
 
     public void upgradeSystem(Version version) {
+        controllerTester().configServer().setDefaultVersion(version);
         updateVersionStatus(version);
         upgrader().maintain();
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
index 286db864c22..4c935747ac6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -181,7 +181,7 @@ public class FailureRedeployerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.141.117");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
@@ -200,7 +200,7 @@ public class FailureRedeployerTest {
 
         // New version is released
         version = Version.fromString("6.142.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
@@ -237,7 +237,7 @@ public class FailureRedeployerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.42.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
@@ -262,7 +262,7 @@ public class FailureRedeployerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.42.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
new file mode 100644
index 00000000000..feaf1289853
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java
@@ -0,0 +1,53 @@
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.component.Version;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.versions.DeploymentStatistics;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
+import org.junit.Test;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class VersionStatusSerializerTest {
+
+    @Test
+    public void testSerialization() throws Exception {
+        List<VespaVersion> vespaVersions = new ArrayList<>();
+        DeploymentStatistics statistics = new DeploymentStatistics(
+                Version.fromString("5.0"),
+                Arrays.asList(ApplicationId.from("tenant1", "failing1", "default")),
+                Arrays.asList(ApplicationId.from("tenant2", "success1", "default"),
+                              ApplicationId.from("tenant2", "success2", "default"))
+        );
+        vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false,
+                                           Arrays.asList("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
+        vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true,
+                                           Arrays.asList("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal));
+        VersionStatus status = new VersionStatus(vespaVersions);
+        VersionStatusSerializer serializer = new VersionStatusSerializer();
+        VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status));
+
+        assertEquals(status.versions().size(), deserialized.versions().size());
+        for (int i = 0; i < status.versions().size(); i++) {
+            VespaVersion a = status.versions().get(i);
+            VespaVersion b = deserialized.versions().get(i);
+            assertEquals(a.releaseCommit(), b.releaseCommit());
+            assertEquals(a.releasedAt(), b.releasedAt());
+            assertEquals(a.isCurrentSystemVersion(), b.isCurrentSystemVersion());
+            assertEquals(a.statistics(), b.statistics());
+            assertEquals(a.configServerHostnames(), b.configServerHostnames());
+            assertEquals(a.confidence(), b.confidence());
+        }
+
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index c002c7fb24a..55a4b46f4a7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -85,7 +85,8 @@ public class DeploymentApiTest extends ControllerContainerTest {
                                            version.releasedAt(), 
                                            version.isCurrentSystemVersion(), 
                                            ImmutableSet.of("config1.test", "config2.test"),
-                                           controller);
+                                           VespaVersion.confidenceFrom(version.statistics(), controller, version.releasedAt())
+                );
             censored.add(version);
         }
         return new VersionStatus(censored);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index c2e83373cf7..00bd1ed8208 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -2,7 +2,7 @@
   "versions": [
     {
       "version": "(ignore)",
-      "confidence": "normal",
+      "confidence": "high",
       "commit": "(ignore)",
       "date": 0,
       "controllerVersion": false,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 7bbbf8f0499..17935906186 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -6,6 +6,7 @@ import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
@@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
 import org.junit.Test;
 
 import java.net.URI;
@@ -46,7 +48,7 @@ public class VersionStatusTest {
     public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() {
         ControllerTester tester = new ControllerTester();
         Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1);
-        tester.configServer().setDefaultConfigServerVersion(largerThanCurrent);
+        tester.configServer().setDefaultVersion(largerThanCurrent);
         VersionStatus versionStatus = VersionStatus.compute(tester.controller());
         assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber());
     }
@@ -55,7 +57,7 @@ public class VersionStatusTest {
     public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException {
         ControllerTester tester = new ControllerTester();
         Version oldest = new Version(5);
-        tester.configServer().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest);
+        tester.configServer().versions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest);
         VersionStatus versionStatus = VersionStatus.compute(tester.controller());
         assertEquals(oldest, versionStatus.systemVersion().get().versionNumber());
     }
@@ -70,7 +72,6 @@ public class VersionStatusTest {
                 .region("us-east-3")
                 .build();
 
-        // Application versions which are older than the current version
         Version version1 = new Version("5.1");
         Version version2 = new Version("5.2");
         tester.upgradeSystem(version1);
@@ -90,14 +91,9 @@ public class VersionStatusTest {
         // - app3 is in production on version1, but then fails in staging test on version2
         tester.completeUpgradeWithError(app3, version2, applicationPackage, stagingTest);
 
-        VersionStatus versionStatus = VersionStatus.compute(tester.controller());
-        List<VespaVersion> versions = versionStatus.versions();
-        assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size());
-
-        VespaVersion v0 = versions.get(2);
-        assertEquals(tester.configServer().getDefaultConfigServerVersion(), v0.versionNumber());
-        assertEquals(0, v0.statistics().failing().size());
-        assertEquals(0, v0.statistics().production().size());
+        tester.updateVersionStatus();
+        List<VespaVersion> versions = tester.controller().versionStatus().versions();
+        assertEquals("The two versions above exist", 2, versions.size());
 
         VespaVersion v1 = versions.get(0);
         assertEquals(version1, v1.versionNumber());
@@ -116,11 +112,6 @@ public class VersionStatusTest {
         // Only one application is on v2 in at least one zone
         assertEquals(1, v2.statistics().production().size());
         assertTrue(v2.statistics().production().contains(app2.id()));
-
-        VespaVersion v3 = versions.get(3);
-        assertEquals(Vtag.currentVersion, v3.versionNumber());
-        assertEquals(0, v3.statistics().failing().size());
-        assertEquals(0, v3.statistics().production().size());
     }
     
     @Test
@@ -130,7 +121,7 @@ public class VersionStatusTest {
         Version version0 = new Version("5.0");
         tester.upgradeSystem(version0);
 
-        // Setup applications
+        // Setup applications - all running on version0
         Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
         Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
         Application canary2 = tester.createAndDeploy("canary2", 3, "canary");
@@ -146,8 +137,7 @@ public class VersionStatusTest {
         Application default9 = tester.createAndDeploy("default9", 13, "default");
         Application conservative0 = tester.createAndDeploy("conservative1", 14, "conservative");
 
-
-        // The following applications should not affect confidence calculation:
+        // Applications that do not affect confidence calculation:
 
         // Application without deployment
         Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L);
@@ -157,30 +147,40 @@ public class VersionStatusTest {
                                                                            "ignored1",
                                                                            "default-pr42", 1000);
 
+        assertEquals("All applications running on this version: High",
+                     Confidence.high, confidence(tester.controller(), version0));
+
+        // New version is released
         Version version1 = new Version("5.1");
-        Version version2 = new Version("5.2");
         tester.upgradeSystem(version1);
 
         // Canaries upgrade to new versions and fail
         tester.completeUpgrade(canary0, version1, "canary");
         tester.completeUpgradeWithError(canary1, version1, "canary", productionUsWest1);
-        tester.upgradeSystem(version2);
-        tester.completeUpgrade(canary2, version2, "canary");
-
-        VersionStatus versionStatus = VersionStatus.compute(tester.controller());
-        List<VespaVersion> versions = versionStatus.versions();
-        
+        tester.updateVersionStatus();
         assertEquals("One canary failed: Broken",
-                     VespaVersion.Confidence.broken, confidence(versions, version1));
-        assertEquals("Nothing has failed but not all canaries has deployed: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version2));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     Confidence.broken, confidence(tester.controller(), version1));
 
-        // All canaries are upgraded to version2 which raises confidence to normal and more apps upgrade
+        // New version is released
+        Version version2 = new Version("5.2");
+        tester.upgradeSystem(version2);
+        assertEquals("Confidence defaults to low for version with no applications",
+                     Confidence.low, confidence(tester.controller(), version2));
+
+        // All canaries upgrade successfully
         tester.completeUpgrade(canary0, version2, "canary");
         tester.completeUpgrade(canary1, version2, "canary");
+
+        assertEquals("Confidence for remains unchanged for version1: Broken",
+                     Confidence.broken, confidence(tester.controller(), version1));
+        assertEquals("Nothing has failed but not all canaries have upgraded: Low",
+                     Confidence.low, confidence(tester.controller(), version2));
+
+        // Remaining canary upgrades to version2 which raises confidence to normal and more apps upgrade
+        tester.completeUpgrade(canary2, version2, "canary");
         tester.upgradeSystem(version2);
+        assertEquals("Canaries have upgraded: Normal",
+                     Confidence.normal, confidence(tester.controller(), version2));
         tester.completeUpgrade(default0, version2, "default");
         tester.completeUpgrade(default1, version2, "default");
         tester.completeUpgrade(default2, version2, "default");
@@ -189,29 +189,27 @@ public class VersionStatusTest {
         tester.completeUpgrade(default5, version2, "default");
         tester.completeUpgrade(default6, version2, "default");
         tester.completeUpgrade(default7, version2, "default");
+        tester.updateVersionStatus();
 
-        versionStatus = VersionStatus.compute(tester.controller());
-        versions = versionStatus.versions();
+        // Remember confidence across restart
+        tester.restartController();
 
-        assertEquals("No deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version0));
+        assertEquals("Confidence remains unchanged for version0: High",
+                     Confidence.high, confidence(tester.controller(), version0));
         assertEquals("All canaries deployed + < 90% of defaults: Normal",
-                     VespaVersion.Confidence.normal, confidence(versions, version2));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     Confidence.normal, confidence(tester.controller(), version2));
+        assertTrue("Status for version without applications is removed",
+                   tester.controller().versionStatus().versions().stream()
+                           .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1)));
 
         // Another default application upgrades, raising confidence to high
         tester.completeUpgrade(default8, version2, "default");
+        tester.updateVersionStatus();
 
-        versionStatus = VersionStatus.compute(tester.controller());
-        versions = versionStatus.versions();
-
-        assertEquals("No deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version0));
+        assertEquals("Confidence remains unchanged for version0: High",
+                     Confidence.high, confidence(tester.controller(), version0));
         assertEquals("90% of defaults deployed successfully: High",
-                     VespaVersion.Confidence.high, confidence(versions, version2));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     VespaVersion.Confidence.high, confidence(tester.controller(), version2));
 
         // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version
         // as broken
@@ -225,16 +223,14 @@ public class VersionStatusTest {
         tester.completeUpgradeWithError(default1, version3, "default", stagingTest);
         tester.completeUpgradeWithError(default2, version3, "default", stagingTest);
         tester.completeUpgradeWithError(default9, version3, "default", stagingTest);
+        tester.updateVersionStatus();
 
-        versionStatus = VersionStatus.compute(tester.controller());
-        versions = versionStatus.versions();
-
-        assertEquals("No deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version0));
+        assertEquals("Confidence remains unchanged for version0: High",
+                     Confidence.high, confidence(tester.controller(), version0));
+        assertEquals("Confidence remains unchanged for version2: High",
+                     Confidence.high, confidence(tester.controller(), version2));
         assertEquals("40% of defaults failed: Broken",
-                     VespaVersion.Confidence.broken, confidence(versions, version3));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     VespaVersion.Confidence.broken, confidence(tester.controller(), version3));
     }
 
     @Test
@@ -260,8 +256,8 @@ public class VersionStatusTest {
                    vespaVersions.stream().noneMatch(v -> v.versionNumber().equals(versionWithUnknownTag)));
     }
 
-    private VespaVersion.Confidence confidence(List<VespaVersion> versions, Version version) {
-        return versions.stream()
+    private Confidence confidence(Controller controller, Version version) {
+        return controller.versionStatus().versions().stream()
                 .filter(v -> v.statistics().version().equals(version))
                 .findFirst()
                 .map(VespaVersion::confidence)