Freeze confidence for previous versions

Once the system upgrades to a new version we're only interested in keeping
confidence for that version up to date. The confidence for previous versions
should be frozen.

This solves several issues:

     * Confidence does not change to 'low' for all other version once canaries
       move to the newest version

     * Applications can keep upgrading to the newest 'normal' or 'high' version
       in case the newest version is either 'low' or 'broken'

7 files changed, 110 insertions, 107 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index ef6cd677b76..c3b2da42861 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -5,9 +5,9 @@ import com.google.common.collect.ImmutableList;
 import com.yahoo.collections.ListMap;
 import com.yahoo.component.Version;
 import com.yahoo.component.Vtag;
-import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
@@ -176,12 +176,29 @@ public class VersionStatus {
                                               Controller controller) {
         GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString());
         Instant releasedAt = Instant.ofEpochMilli(gitSha.commit.author.date.getTime());
+        VespaVersion.Confidence confidence;
+        // Always compute confidence for system version
+        if (isSystemVersion) {
+            confidence = VespaVersion.confidenceFrom(statistics, controller, releasedAt);
+        } else {
+            // Keep existing confidence for non-system versions if already computed
+            confidence = confidenceFor(statistics.version(), controller)
+                    .orElse(VespaVersion.confidenceFrom(statistics, controller, releasedAt));
+        }
         return new VespaVersion(statistics,
                                 gitSha.sha, releasedAt,
                                 isSystemVersion,
                                 configServerHostnames,
-                                VespaVersion.confidenceFrom(statistics, controller, releasedAt)
+                                confidence
         );
     }
 
+    /** Returns the current confidence for the given version */
+    private static Optional<VespaVersion.Confidence> confidenceFor(Version version, Controller controller) {
+        return controller.versionStatus().versions().stream()
+                .filter(v -> version.equals(v.versionNumber()))
+                .map(VespaVersion::confidence)
+                .findFirst();
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
index 9db852374b8..9228e83bbc6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ConfigServerClientMock.java
@@ -7,16 +7,16 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerClient;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeList;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
 import com.yahoo.vespa.hosted.controller.api.rotation.Rotation;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.vespa.serviceview.bindings.ClusterView;
@@ -28,7 +28,6 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -40,15 +39,11 @@ import java.util.UUID;
  */
 public class ConfigServerClientMock extends AbstractComponent implements ConfigServerClient {
 
-    private Map<ApplicationId, byte[]> applicationContent = new HashMap<>();
-    private Map<ApplicationId, String> applicationInstances = new HashMap<>();
-    private Map<ApplicationId, Boolean> applicationActivated = new HashMap<>();
-    private Set<ApplicationId> applicationRestarted = new HashSet<>();
-    private Set<String> hostsExplicitlyRestarted = new HashSet<>();
-    private Map<String, EndpointStatus> endpoints = new HashMap<>();
-
-    private Map<URI, Version> configServerVersions = new HashMap<>();
-    private Version defaultConfigServerVersion = new Version(6, 1, 0);
+    private final Map<ApplicationId, String> applicationInstances = new HashMap<>();
+    private final Map<ApplicationId, Boolean> applicationActivated = new HashMap<>();
+    private final Map<String, EndpointStatus> endpoints = new HashMap<>();
+    private final Map<URI, Version> versions = new HashMap<>();
+    private Version defaultVersion = new Version(6, 1, 0);
 
     /** The exception to throw on the next prepare run, or null to continue normally */
     private RuntimeException prepareException = null;
@@ -64,18 +59,27 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
     public Map<ApplicationId, Boolean> activated() {
         return Collections.unmodifiableMap(applicationActivated);
     }
+
+    public void throwOnNextPrepare(RuntimeException prepareException) {
+        this.prepareException = prepareException;
+    }
+
+    /**
+     * Returns the (initially empty) mutable map of config server urls to versions.
+     * This API will return defaultVersion as response to any version(url) call for versions not added to the map.
+     */
+    public Map<URI, Version> versions() {
+        return versions;
+    }
     
     @Override
     public PreparedApplication prepare(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames, Set<Rotation> rotations, byte[] content) {
         lastPrepareVersion = deployOptions.vespaVersion.map(Version::new);
-        
         if (prepareException != null) {
             RuntimeException prepareException = this.prepareException;
             this.prepareException = null;
             throw prepareException;
         }
-
-        applicationContent.put(deployment.applicationId(), content);
         applicationActivated.put(deployment.applicationId(), false);
         applicationInstances.put(deployment.applicationId(), UUID.randomUUID() + ":4080");
 
@@ -111,20 +115,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
         };
     }
 
-    public void throwOnNextPrepare(RuntimeException prepareException) {
-        this.prepareException = prepareException;
-    }
-
-    /**
-     * Returns the (initially empty) mutable map of config server urls to versions.
-     * This API will return defaultConfigserverVersion as response to any version(url) call for versions not added to the map.
-     */
-    public Map<URI, Version> configServerVersions() {
-        return configServerVersions;
-    }
-    
-    public Version getDefaultConfigServerVersion() { return defaultConfigServerVersion; }
-    public void setDefaultConfigServerVersion(Version version) { defaultConfigServerVersion = version; }
+    /** Set the default config server version */
+    public void setDefaultVersion(Version version) { this.defaultVersion = version; }
     
     @Override
     public List<String> getNodeQueryHost(DeploymentId deployment, String type) {
@@ -137,16 +129,11 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
 
     @Override
     public void restart(DeploymentId deployment, Optional<Hostname> hostname) {
-        applicationRestarted.add(deployment.applicationId());
-        if (hostname.isPresent()) {
-            hostsExplicitlyRestarted.add(hostname.get().id());
-        }
     }
 
     @Override
     public void deactivate(DeploymentId deployment) {
         applicationActivated.remove(deployment.applicationId());
-        applicationContent.remove(deployment.applicationId());
         applicationInstances.remove(deployment.applicationId());
     }
 
@@ -197,7 +184,7 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
     
     @Override
     public Version version(URI configServerURI) {
-        return configServerVersions.getOrDefault(configServerURI, defaultConfigServerVersion);
+        return versions.getOrDefault(configServerURI, defaultVersion);
     }
 
     @Override
@@ -207,10 +194,8 @@ public class ConfigServerClientMock extends AbstractComponent implements ConfigS
 
     @Override
     public EndpointStatus getGlobalRotationStatus(DeploymentId deployment, String endpoint) {
-        EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757l);
-        return endpoints.containsKey(endpoint)
-                ? endpoints.get(endpoint)
-                : result;
+        EndpointStatus result = new EndpointStatus(EndpointStatus.Status.in, "", "", 1497618757L);
+        return endpoints.getOrDefault(endpoint, result);
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 105e654e48a..b193fd7ef41 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -545,7 +545,7 @@ public class ControllerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.141.117");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
@@ -575,7 +575,7 @@ public class ControllerTest {
 
         // New version is released
         version = Version.fromString("6.142.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index be14947de2b..e45166b1c16 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -90,12 +90,17 @@ public class DeploymentTester {
                 .filter(c -> c instanceof Change.VersionChange)
                 .map(Change.VersionChange.class::cast);
     }
+
+    public void updateVersionStatus() {
+        controller().updateVersionStatus(VersionStatus.compute(controller(), tester.controller().systemVersion()));
+    }
     
     public void updateVersionStatus(Version currentVersion) {
         controller().updateVersionStatus(VersionStatus.compute(controller(), currentVersion));
     }
 
     public void upgradeSystem(Version version) {
+        controllerTester().configServer().setDefaultVersion(version);
         updateVersionStatus(version);
         upgrader().maintain();
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
index 286db864c22..4c935747ac6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -181,7 +181,7 @@ public class FailureRedeployerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.141.117");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
@@ -200,7 +200,7 @@ public class FailureRedeployerTest {
 
         // New version is released
         version = Version.fromString("6.142.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
         tester.upgrader().maintain();
@@ -237,7 +237,7 @@ public class FailureRedeployerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.42.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
@@ -262,7 +262,7 @@ public class FailureRedeployerTest {
 
         // Current system version, matches version in test data
         Version version = Version.fromString("6.42.1");
-        tester.configServer().setDefaultConfigServerVersion(version);
+        tester.configServer().setDefaultVersion(version);
         tester.updateVersionStatus(version);
         assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index c2e83373cf7..00bd1ed8208 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -2,7 +2,7 @@
   "versions": [
     {
       "version": "(ignore)",
-      "confidence": "normal",
+      "confidence": "high",
       "commit": "(ignore)",
       "date": 0,
       "controllerVersion": false,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 7bbbf8f0499..17935906186 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -6,6 +6,7 @@ import com.yahoo.component.Vtag;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
+import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
@@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
 import org.junit.Test;
 
 import java.net.URI;
@@ -46,7 +48,7 @@ public class VersionStatusTest {
     public void testSystemVersionIsControllerVersionIfConfigserversAreNewer() {
         ControllerTester tester = new ControllerTester();
         Version largerThanCurrent = new Version(Vtag.currentVersion.getMajor() + 1);
-        tester.configServer().setDefaultConfigServerVersion(largerThanCurrent);
+        tester.configServer().setDefaultVersion(largerThanCurrent);
         VersionStatus versionStatus = VersionStatus.compute(tester.controller());
         assertEquals(Vtag.currentVersion, versionStatus.systemVersion().get().versionNumber());
     }
@@ -55,7 +57,7 @@ public class VersionStatusTest {
     public void testSystemVersionIsVersionOfOldestConfigServer() throws URISyntaxException {
         ControllerTester tester = new ControllerTester();
         Version oldest = new Version(5);
-        tester.configServer().configServerVersions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest);
+        tester.configServer().versions().put(new URI("http://cfg.prod.corp-us-east-1.test"), oldest);
         VersionStatus versionStatus = VersionStatus.compute(tester.controller());
         assertEquals(oldest, versionStatus.systemVersion().get().versionNumber());
     }
@@ -70,7 +72,6 @@ public class VersionStatusTest {
                 .region("us-east-3")
                 .build();
 
-        // Application versions which are older than the current version
         Version version1 = new Version("5.1");
         Version version2 = new Version("5.2");
         tester.upgradeSystem(version1);
@@ -90,14 +91,9 @@ public class VersionStatusTest {
         // - app3 is in production on version1, but then fails in staging test on version2
         tester.completeUpgradeWithError(app3, version2, applicationPackage, stagingTest);
 
-        VersionStatus versionStatus = VersionStatus.compute(tester.controller());
-        List<VespaVersion> versions = versionStatus.versions();
-        assertEquals("The version of this controller, the default config server version, plus the two versions above exist", 4, versions.size());
-
-        VespaVersion v0 = versions.get(2);
-        assertEquals(tester.configServer().getDefaultConfigServerVersion(), v0.versionNumber());
-        assertEquals(0, v0.statistics().failing().size());
-        assertEquals(0, v0.statistics().production().size());
+        tester.updateVersionStatus();
+        List<VespaVersion> versions = tester.controller().versionStatus().versions();
+        assertEquals("The two versions above exist", 2, versions.size());
 
         VespaVersion v1 = versions.get(0);
         assertEquals(version1, v1.versionNumber());
@@ -116,11 +112,6 @@ public class VersionStatusTest {
         // Only one application is on v2 in at least one zone
         assertEquals(1, v2.statistics().production().size());
         assertTrue(v2.statistics().production().contains(app2.id()));
-
-        VespaVersion v3 = versions.get(3);
-        assertEquals(Vtag.currentVersion, v3.versionNumber());
-        assertEquals(0, v3.statistics().failing().size());
-        assertEquals(0, v3.statistics().production().size());
     }
     
     @Test
@@ -130,7 +121,7 @@ public class VersionStatusTest {
         Version version0 = new Version("5.0");
         tester.upgradeSystem(version0);
 
-        // Setup applications
+        // Setup applications - all running on version0
         Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
         Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
         Application canary2 = tester.createAndDeploy("canary2", 3, "canary");
@@ -146,8 +137,7 @@ public class VersionStatusTest {
         Application default9 = tester.createAndDeploy("default9", 13, "default");
         Application conservative0 = tester.createAndDeploy("conservative1", 14, "conservative");
 
-
-        // The following applications should not affect confidence calculation:
+        // Applications that do not affect confidence calculation:
 
         // Application without deployment
         Application ignored0 = tester.createApplication("ignored0", "tenant1", 1000, 1000L);
@@ -157,30 +147,40 @@ public class VersionStatusTest {
                                                                            "ignored1",
                                                                            "default-pr42", 1000);
 
+        assertEquals("All applications running on this version: High",
+                     Confidence.high, confidence(tester.controller(), version0));
+
+        // New version is released
         Version version1 = new Version("5.1");
-        Version version2 = new Version("5.2");
         tester.upgradeSystem(version1);
 
         // Canaries upgrade to new versions and fail
         tester.completeUpgrade(canary0, version1, "canary");
         tester.completeUpgradeWithError(canary1, version1, "canary", productionUsWest1);
-        tester.upgradeSystem(version2);
-        tester.completeUpgrade(canary2, version2, "canary");
-
-        VersionStatus versionStatus = VersionStatus.compute(tester.controller());
-        List<VespaVersion> versions = versionStatus.versions();
-        
+        tester.updateVersionStatus();
         assertEquals("One canary failed: Broken",
-                     VespaVersion.Confidence.broken, confidence(versions, version1));
-        assertEquals("Nothing has failed but not all canaries has deployed: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version2));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     Confidence.broken, confidence(tester.controller(), version1));
 
-        // All canaries are upgraded to version2 which raises confidence to normal and more apps upgrade
+        // New version is released
+        Version version2 = new Version("5.2");
+        tester.upgradeSystem(version2);
+        assertEquals("Confidence defaults to low for version with no applications",
+                     Confidence.low, confidence(tester.controller(), version2));
+
+        // All canaries upgrade successfully
         tester.completeUpgrade(canary0, version2, "canary");
         tester.completeUpgrade(canary1, version2, "canary");
+
+        assertEquals("Confidence for remains unchanged for version1: Broken",
+                     Confidence.broken, confidence(tester.controller(), version1));
+        assertEquals("Nothing has failed but not all canaries have upgraded: Low",
+                     Confidence.low, confidence(tester.controller(), version2));
+
+        // Remaining canary upgrades to version2 which raises confidence to normal and more apps upgrade
+        tester.completeUpgrade(canary2, version2, "canary");
         tester.upgradeSystem(version2);
+        assertEquals("Canaries have upgraded: Normal",
+                     Confidence.normal, confidence(tester.controller(), version2));
         tester.completeUpgrade(default0, version2, "default");
         tester.completeUpgrade(default1, version2, "default");
         tester.completeUpgrade(default2, version2, "default");
@@ -189,29 +189,27 @@ public class VersionStatusTest {
         tester.completeUpgrade(default5, version2, "default");
         tester.completeUpgrade(default6, version2, "default");
         tester.completeUpgrade(default7, version2, "default");
+        tester.updateVersionStatus();
 
-        versionStatus = VersionStatus.compute(tester.controller());
-        versions = versionStatus.versions();
+        // Remember confidence across restart
+        tester.restartController();
 
-        assertEquals("No deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version0));
+        assertEquals("Confidence remains unchanged for version0: High",
+                     Confidence.high, confidence(tester.controller(), version0));
         assertEquals("All canaries deployed + < 90% of defaults: Normal",
-                     VespaVersion.Confidence.normal, confidence(versions, version2));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     Confidence.normal, confidence(tester.controller(), version2));
+        assertTrue("Status for version without applications is removed",
+                   tester.controller().versionStatus().versions().stream()
+                           .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1)));
 
         // Another default application upgrades, raising confidence to high
         tester.completeUpgrade(default8, version2, "default");
+        tester.updateVersionStatus();
 
-        versionStatus = VersionStatus.compute(tester.controller());
-        versions = versionStatus.versions();
-
-        assertEquals("No deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version0));
+        assertEquals("Confidence remains unchanged for version0: High",
+                     Confidence.high, confidence(tester.controller(), version0));
         assertEquals("90% of defaults deployed successfully: High",
-                     VespaVersion.Confidence.high, confidence(versions, version2));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     VespaVersion.Confidence.high, confidence(tester.controller(), version2));
 
         // A new version is released, all canaries upgrade successfully, but enough "default" apps fail to mark version
         // as broken
@@ -225,16 +223,14 @@ public class VersionStatusTest {
         tester.completeUpgradeWithError(default1, version3, "default", stagingTest);
         tester.completeUpgradeWithError(default2, version3, "default", stagingTest);
         tester.completeUpgradeWithError(default9, version3, "default", stagingTest);
+        tester.updateVersionStatus();
 
-        versionStatus = VersionStatus.compute(tester.controller());
-        versions = versionStatus.versions();
-
-        assertEquals("No deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, version0));
+        assertEquals("Confidence remains unchanged for version0: High",
+                     Confidence.high, confidence(tester.controller(), version0));
+        assertEquals("Confidence remains unchanged for version2: High",
+                     Confidence.high, confidence(tester.controller(), version2));
         assertEquals("40% of defaults failed: Broken",
-                     VespaVersion.Confidence.broken, confidence(versions, version3));
-        assertEquals("Current version of this - no deployments: Low",
-                     VespaVersion.Confidence.low, confidence(versions, Vtag.currentVersion));
+                     VespaVersion.Confidence.broken, confidence(tester.controller(), version3));
     }
 
     @Test
@@ -260,8 +256,8 @@ public class VersionStatusTest {
                    vespaVersions.stream().noneMatch(v -> v.versionNumber().equals(versionWithUnknownTag)));
     }
 
-    private VespaVersion.Confidence confidence(List<VespaVersion> versions, Version version) {
-        return versions.stream()
+    private Confidence confidence(Controller controller, Version version) {
+        return controller.versionStatus().versions().stream()
                 .filter(v -> v.statistics().version().equals(version))
                 .findFirst()
                 .map(VespaVersion::confidence)