Merge pull request #10627 from vespa-engine/mpolden/time-restricted-confidence-change

Restrict confidence change to a fixed time window

7 files changed, 174 insertions, 51 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index de7f12efcaf..322d0aa8d30 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
@@ -159,4 +159,5 @@ public class Upgrader extends Maintainer {
     public void removeConfidenceOverride(Version version) {
         controller().removeConfidenceOverride(version::equals);
     }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 63d470a5b1d..87ae2538ad6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -40,6 +40,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobEr
  * This is immutable.
  * 
  * @author bratseth
+ * @author mpolden
  */
 public class VersionStatus {
 
@@ -245,22 +246,32 @@ public class VersionStatus {
         var isSystemVersion = statistics.version().equals(systemVersion);
         var isControllerVersion = statistics.version().equals(controllerVersion.version());
         var confidence = controller.curator().readConfidenceOverrides().get(statistics.version());
-        // Compute confidence if there's no override
-        if (confidence == null) {
-            if (isSystemVersion || isControllerVersion) { // Always compute confidence for system and controller
+        var confidenceIsOverridden = confidence != null;
+        var previousStatus = controller.versionStatus().version(statistics.version());
+
+        // Compute confidence
+        if (!confidenceIsOverridden) {
+            // Always compute confidence for system and controller
+            if (isSystemVersion || isControllerVersion) {
                 confidence = VespaVersion.confidenceFrom(statistics, controller);
-            } else { // This is an older version so we preserve the existing confidence, if any
-                confidence = confidenceFor(statistics.version(), controller)
-                        .orElseGet(() -> VespaVersion.confidenceFrom(statistics, controller));
+            } else {
+                // This is an older version so we preserve the existing confidence, if any
+                confidence = getOrUpdateConfidence(statistics, controller);
             }
         }
+
         // Preserve existing commit details if we've previously computed status for this version
         var commitSha = controllerVersion.commitSha();
         var commitDate = controllerVersion.commitDate();
-        var previousStatus = controller.versionStatus().version(statistics.version());
         if (previousStatus != null) {
             commitSha = previousStatus.releaseCommit();
             commitDate = previousStatus.committedAt();
+
+            // Keep existing confidence if we cannot raise it at this moment in time
+            if (!confidenceIsOverridden &&
+                !previousStatus.confidence().canChangeTo(confidence, controller.clock().instant())) {
+                confidence = previousStatus.confidence();
+            }
         }
         return new VespaVersion(statistics,
                                 commitSha,
@@ -269,16 +280,20 @@ public class VersionStatus {
                                 isSystemVersion,
                                 isReleased,
                                 configServerHostnames,
-                                confidence
-        );
+                                confidence);
     }
 
-    /** Returns the current confidence for the given version */
-    private static Optional<VespaVersion.Confidence> confidenceFor(Version version, Controller controller) {
+    /**
+     * Calculate confidence from given deployment statistics.
+     *
+     * @return previously calculated confidence for this version. If none exists, a new confidence will be calculated.
+     */
+    private static VespaVersion.Confidence getOrUpdateConfidence(DeploymentStatistics statistics, Controller controller) {
         return controller.versionStatus().versions().stream()
-                         .filter(v -> version.equals(v.versionNumber()))
+                         .filter(v -> statistics.version().equals(v.versionNumber()))
                          .map(VespaVersion::confidence)
-                         .findFirst();
+                         .findFirst()
+                         .orElseGet(() -> VespaVersion.confidenceFrom(statistics, controller));
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index 117ce80adaa..be277bc2262 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
 import com.google.common.collect.ImmutableSet;
@@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 
 import java.time.Instant;
+import java.time.ZoneOffset;
 import java.util.Collection;
 import java.util.Set;
 
@@ -148,6 +149,16 @@ public class VespaVersion implements Comparable<VespaVersion> {
             return this.compareTo(other) >= 0;
         }
 
+        /** Returns true if this can be changed to target at given instant */
+        public boolean canChangeTo(Confidence target, Instant instant) {
+            if (this.equalOrHigherThan(normal)) return true; // Confidence can always change from >= normal
+            if (!target.equalOrHigherThan(normal)) return true; // Confidence can always change to < normal
+
+            var hourOfDay = instant.atZone(ZoneOffset.UTC).getHour();
+            // Confidence can only be raised between 05:00:00 and 11:59:59 UTC
+            return hourOfDay >= 5 && hourOfDay <= 11;
+        }
+
     }
 
     private static boolean nonCanaryApplicationsBroken(Version version,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 04603245af6..cc5e22d775b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -29,6 +29,8 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 
 import java.time.Duration;
 import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
 import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
@@ -75,6 +77,19 @@ public class DeploymentTester {
         this.nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), Duration.ofHours(12),
                                                                new JobControl(tester.controller().curator()),
                                                                Integer.MAX_VALUE);
+        atHourOfDay(5); // Set hour of day which always allows confidence to change
+    }
+
+    public DeploymentTester atHourOfDay(int hour) {
+        var dateTime = tester.clock().instant().atZone(ZoneOffset.UTC);
+        return at(LocalDateTime.of(dateTime.getYear(), dateTime.getMonth(), dateTime.getDayOfMonth(), hour,
+                                   dateTime.getMinute(), dateTime.getSecond())
+                               .toInstant(ZoneOffset.UTC));
+    }
+
+    public DeploymentTester at(Instant instant) {
+        tester.clock().setInstant(instant);
+        return this;
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -165,6 +180,11 @@ public class DeploymentTester {
 
     public void restartController() { tester.createNewController(); }
 
+    public int hourOfDayAfter(Duration duration) {
+        tester.clock().advance(duration);
+        return tester.controller().clock().instant().atOffset(ZoneOffset.UTC).getHour();
+    }
+
     /** Notify the controller about a job completing */
     public BuildJob jobCompletion(JobType job) {
         return new BuildJob(this::notifyJobCompletion, tester.serviceRegistry().artifactRepositoryMock()).type(job);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 0d8d32299ae..f8e7026d9d5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -3,10 +3,9 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.Slime;
-import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
@@ -17,7 +16,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
@@ -442,8 +440,8 @@ public class DeploymentTriggerTest {
 
     @Test
     public void testBlockRevisionChange() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:30:00.00Z")); // Tuesday, 17:30
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Tuesday, 17:30
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-26T17:30:00.00Z"));
         ReadyJobsTrigger readyJobsTrigger = new ReadyJobsTrigger(tester.controller(),
                                                                  Duration.ofHours(1),
                                                                  new JobControl(tester.controllerTester().curator()));
@@ -501,8 +499,8 @@ public class DeploymentTriggerTest {
 
     @Test
     public void testCompletionOfPartOfChangeDuringBlockWindow() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:30:00.00Z")); // Tuesday, 17:30
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Tuesday, 17:30
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-26T17:30:00.00Z"));
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .blockChange(true, true, "tue", "18", "UTC")
                 .region("us-west-1")
@@ -522,7 +520,7 @@ public class DeploymentTriggerTest {
         tester.deployAndNotify(application, applicationPackage, true, systemTest);
 
         // Entering block window will keep the outstanding change in place.
-        clock.advance(Duration.ofHours(1));
+        tester.clock().advance(Duration.ofHours(1));
         tester.outstandingChangeDeployer().run();
         tester.deployAndNotify(application, applicationPackage, true, productionUsWest1);
         assertEquals(BuildJob.defaultBuildNumber, tester.application(application.id()).deploymentJobs().jobStatus()
@@ -542,7 +540,7 @@ public class DeploymentTriggerTest {
 
         tester.deployAndNotify(application, applicationPackage, true, stagingTest);
         tester.deployAndNotify(application, applicationPackage, true, systemTest);
-        clock.advance(Duration.ofHours(1));
+        tester.clock().advance(Duration.ofHours(1));
         tester.outstandingChangeDeployer().run();
         assertTrue(tester.application(application.id()).change().hasTargets());
         assertFalse(tester.application(application.id()).outstandingChange().hasTargets());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index 484516d7cd7..ab4d4987d6a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -1,14 +1,13 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
-import com.yahoo.test.ManualClock;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
@@ -43,11 +42,10 @@ import static org.junit.Assert.assertTrue;
  */
 public class UpgraderTest {
 
-    private final DeploymentTester tester = new DeploymentTester();
-
     @Test
     public void testUpgrading() {
         // --- Setup
+        DeploymentTester tester = new DeploymentTester();
         Version version0 = Version.fromString("6.2");
         tester.upgradeSystem(version0);
 
@@ -260,6 +258,7 @@ public class UpgraderTest {
     @Test
     public void testUpgradingToVersionWhichBreaksSomeNonCanaries() {
         // --- Setup
+        DeploymentTester tester = new DeploymentTester();
         tester.upgrader().maintain();
         tester.triggerUntilQuiescence();
         assertEquals("No system version: Nothing to do", 0, tester.buildService().jobs().size());
@@ -330,6 +329,7 @@ public class UpgraderTest {
 
     @Test
     public void testDeploymentAlreadyInProgressForUpgrade() {
+        DeploymentTester tester = new DeploymentTester();
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .upgradePolicy("canary")
                 .environment(Environment.prod)
@@ -384,6 +384,7 @@ public class UpgraderTest {
 
     @Test
     public void testUpgradeCancelledWithDeploymentInProgress() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -449,6 +450,7 @@ public class UpgraderTest {
      */
     @Test
     public void testVersionIsBrokenAfterAZoneIsLive() {
+        DeploymentTester tester = new DeploymentTester();
         Version v0 = Version.fromString("6.2");
         tester.upgradeSystem(v0);
 
@@ -532,6 +534,7 @@ public class UpgraderTest {
 
     @Test
     public void testConfidenceIgnoresFailingApplicationChanges() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -585,8 +588,8 @@ public class UpgraderTest {
 
     @Test
     public void testBlockVersionChange() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-26T18:00:00.00Z")); // Tuesday, 18:00
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Tuesday, 18:00
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-26T18:00:00.00Z"));
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -625,8 +628,8 @@ public class UpgraderTest {
 
     @Test
     public void testBlockVersionChangeHalfwayThough() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Tuesday, 17:00
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-26T17:00:00.00Z"));
 
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
@@ -651,7 +654,7 @@ public class UpgraderTest {
         tester.triggerUntilQuiescence();
         tester.deployAndNotify(app, applicationPackage, true, systemTest);
         tester.deployAndNotify(app, applicationPackage, true, stagingTest);
-        clock.advance(Duration.ofHours(1)); // Entering block window after prod job is triggered
+        tester.clock().advance(Duration.ofHours(1)); // Entering block window after prod job is triggered
         tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
         assertEquals(1, tester.buildService().jobs().size()); // Next job triggered because upgrade is already rolling out.
 
@@ -662,8 +665,8 @@ public class UpgraderTest {
 
     @Test
     public void testBlockVersionChangeHalfwayThoughThenNewRevision() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-29T16:00:00.00Z")); // Friday, 16:00
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Friday, 16:00
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-29T16:00:00.00Z"));
 
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
@@ -688,7 +691,7 @@ public class UpgraderTest {
         tester.triggerUntilQuiescence();
         tester.deployAndNotify(app, applicationPackage, true, systemTest);
         tester.deployAndNotify(app, applicationPackage, true, stagingTest);
-        clock.advance(Duration.ofHours(1)); // Entering block window after prod job is triggered
+        tester.clock().advance(Duration.ofHours(1)); // Entering block window after prod job is triggered
         tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
         assertEquals(1, tester.buildService().jobs().size()); // Next job triggered, as upgrade is already in progress.
         tester.deployAndNotify(app, applicationPackage, false, productionUsCentral1); // us-central-1 fails, permitting a new revision.
@@ -739,6 +742,7 @@ public class UpgraderTest {
 
     @Test
     public void testReschedulesUpgradeAfterTimeout() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -842,6 +846,7 @@ public class UpgraderTest {
 
     @Test
     public void testThrottlesUpgrades() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -893,6 +898,7 @@ public class UpgraderTest {
 
     @Test
     public void testPinningMajorVersionInDeploymentXml() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -927,6 +933,7 @@ public class UpgraderTest {
 
     @Test
     public void testPinningMajorVersionInApplication() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -962,6 +969,7 @@ public class UpgraderTest {
 
     @Test
     public void testPinningMajorVersionInUpgrader() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -1017,6 +1025,7 @@ public class UpgraderTest {
 
     @Test
     public void testAllowApplicationChangeDuringFailingUpgrade() {
+        DeploymentTester tester = new DeploymentTester();
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
 
@@ -1063,8 +1072,8 @@ public class UpgraderTest {
 
     @Test
     public void testBlockRevisionChangeHalfwayThoughThenUpgrade() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00.
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Tuesday, 17:00.
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-26T17:00:00.00Z"));
 
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
@@ -1087,7 +1096,7 @@ public class UpgraderTest {
         tester.triggerUntilQuiescence();
         tester.deployAndNotify(app, applicationPackage, true, systemTest);
         tester.deployAndNotify(app, applicationPackage, true, stagingTest);
-        clock.advance(Duration.ofHours(1)); // Entering block window after prod job is triggered.
+        tester.clock().advance(Duration.ofHours(1)); // Entering block window after prod job is triggered.
         tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
         assertEquals(1, tester.buildService().jobs().size()); // Next job triggered in spite of block, because it is already rolling out.
 
@@ -1115,8 +1124,8 @@ public class UpgraderTest {
 
     @Test
     public void testBlockRevisionChangeHalfwayThoughThenNewRevision() {
-        ManualClock clock = new ManualClock(Instant.parse("2017-09-26T17:00:00.00Z")); // Tuesday, 17:00.
-        DeploymentTester tester = new DeploymentTester(new ControllerTester(clock));
+        // Tuesday, 17:00.
+        DeploymentTester tester = new DeploymentTester().at(Instant.parse("2017-09-26T17:00:00.00Z"));
 
         Version version = Version.fromString("6.2");
         tester.upgradeSystem(version);
@@ -1139,7 +1148,7 @@ public class UpgraderTest {
         tester.triggerUntilQuiescence();
         tester.deployAndNotify(app, applicationPackage, true, systemTest);
         tester.deployAndNotify(app, applicationPackage, true, stagingTest);
-        clock.advance(Duration.ofHours(1)); // Entering block window after prod job is triggered.
+        tester.clock().advance(Duration.ofHours(1)); // Entering block window after prod job is triggered.
         tester.deployAndNotify(app, applicationPackage, true, productionUsWest1);
         assertEquals(1, tester.buildService().jobs().size());
 
@@ -1156,7 +1165,7 @@ public class UpgraderTest {
 
         tester.outstandingChangeDeployer().run();
         assertFalse(tester.application(app.id()).change().hasTargets());
-        clock.advance(Duration.ofHours(2));
+        tester.clock().advance(Duration.ofHours(2));
 
         tester.outstandingChangeDeployer().run();
         assertTrue(tester.application(app.id()).change().hasTargets());
@@ -1169,6 +1178,7 @@ public class UpgraderTest {
 
     @Test
     public void testPinning() {
+        DeploymentTester tester = new DeploymentTester();
         Version version0 = Version.fromString("6.2");
         tester.upgradeSystem(version0);
 
@@ -1272,6 +1282,7 @@ public class UpgraderTest {
 
     @Test
     public void testsEachUpgradeCombinationWithFailingDeployments() {
+        DeploymentTester tester = new DeploymentTester();
         Application application = tester.createApplication("app1", "tenant1", 1, 1L);
         Supplier<Application> app = () -> tester.application(application.id());
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 7a57ebb65dc..94201832adb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.versions;
 
 import com.google.common.collect.ImmutableSet;
@@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
 import org.junit.Test;
 
+import java.time.Duration;
 import java.time.Instant;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -31,6 +32,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -58,11 +60,11 @@ public class VersionStatusTest {
 
     @Test
     public void testSystemVersionIsVersionOfOldestConfigServer() {
-        ControllerTester tester = new ControllerTester();
+        DeploymentTester tester = new DeploymentTester();
         Version version0 = Version.fromString("6.1");
         Version version1 = Version.fromString("6.5");
         // Upgrade some config servers
-        for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) {
+        for (ZoneApi zone : tester.controllerTester().zoneRegistry().zones().all().zones()) {
             for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) {
                 Node upgradedNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), version1, node.wantedVersion());
                 tester.configServer().nodeRepository().putByHostname(zone.getId(), upgradedNode);
@@ -100,7 +102,6 @@ public class VersionStatusTest {
     @Test
     public void testSystemVersionNeverShrinks() {
         DeploymentTester tester = new DeploymentTester();
-
         Version version0 = Version.fromString("6.2");
         tester.upgradeSystem(version0);
         assertEquals(version0, tester.controller().systemVersion());
@@ -170,7 +171,6 @@ public class VersionStatusTest {
     @Test
     public void testVersionConfidence() {
         DeploymentTester tester = new DeploymentTester();
-
         Version version0 = new Version("6.2");
         tester.upgradeSystem(version0);
 
@@ -333,8 +333,7 @@ public class VersionStatusTest {
 
     @Test
     public void testCommitDetailsPreservation() {
-        var tester = new DeploymentTester();
-
+        DeploymentTester tester = new DeploymentTester();
         // Commit details are set for initial version
         var version0 = new Version("6.2");
         var commitSha0 = "badc0ffee";
@@ -362,6 +361,74 @@ public class VersionStatusTest {
         assertEquals(commitDate0, tester.controller().versionStatus().version(version0).committedAt());
     }
 
+    @Test
+    public void testConfidenceChangeRespectsTimeWindow() {
+        DeploymentTester tester = new DeploymentTester();
+        // Canaries and normal application deploys on initial version
+        assertEquals(5, tester.hourOfDayAfter(Duration.ZERO));
+        Version version0 = Version.fromString("7.1");
+        tester.upgradeSystem(version0);
+        Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
+        Application canary1 = tester.createAndDeploy("canary1", 1, "canary");
+        Application default0 = tester.createAndDeploy("default0", 1, "default");
+        tester.computeVersionStatus();
+        assertSame(Confidence.high, tester.controller().versionStatus().version(version0).confidence());
+
+        // System and canary0 is upgraded within allowed time window
+        Version version1 = Version.fromString("7.2");
+        tester.upgradeSystem(version1);
+        tester.completeUpgrade(canary0, version1, "canary");
+        tester.computeVersionStatus();
+        assertSame(Confidence.low, tester.controller().versionStatus().version(version1).confidence());
+
+        // canary1 breaks just outside allowed upgrade window
+        assertEquals(12, tester.hourOfDayAfter(Duration.ofHours(7)));
+        tester.completeUpgradeWithError(canary1, version1, "canary", systemTest);
+        tester.computeVersionStatus();
+        assertSame(Confidence.broken, tester.controller().versionStatus().version(version1).confidence());
+
+        // Second canary is fixed later in the day. All canaries are now fixed, but confidence is not raised as we're
+        // outside the allowed time window
+        assertEquals(20, tester.hourOfDayAfter(Duration.ofHours(8)));
+        tester.completeUpgrade(canary1, version1, "canary");
+        tester.computeVersionStatus();
+        assertSame(Confidence.broken, tester.controller().versionStatus().version(version1).confidence());
+
+        // Early morning arrives, confidence is raised and normal application upgrades
+        assertEquals(5, tester.hourOfDayAfter(Duration.ofHours(9)));
+        tester.computeVersionStatus();
+        assertSame(Confidence.normal, tester.controller().versionStatus().version(version1).confidence());
+        tester.upgrader().maintain();
+        tester.triggerUntilQuiescence();
+        tester.completeUpgrade(default0, version1, "default");
+
+        // Another version is released. System and canaries upgrades late, confidence stays low
+        Version version2 = Version.fromString("7.3");
+        tester.upgradeSystem(version2);
+        assertEquals(14, tester.hourOfDayAfter(Duration.ofHours(9)));
+        tester.completeUpgrade(canary0, version2, "canary");
+        tester.completeUpgrade(canary1, version2, "canary");
+        tester.computeVersionStatus();
+        assertSame(Confidence.low, tester.controller().versionStatus().version(version2).confidence());
+
+        // Confidence override takes precedence over time window constraints
+        tester.upgrader().overrideConfidence(version2, Confidence.normal);
+        tester.computeVersionStatus();
+        assertSame(Confidence.normal, tester.controller().versionStatus().version(version2).confidence());
+        tester.upgrader().overrideConfidence(version2, Confidence.low);
+        tester.computeVersionStatus();
+        assertSame(Confidence.low, tester.controller().versionStatus().version(version2).confidence());
+        tester.upgrader().removeConfidenceOverride(version2);
+
+        // Next morning arrives, confidence is raised and normal application upgrades
+        assertEquals(7, tester.hourOfDayAfter(Duration.ofHours(17)));
+        tester.computeVersionStatus();
+        assertSame(Confidence.normal, tester.controller().versionStatus().version(version2).confidence());
+        tester.upgrader().maintain();
+        tester.triggerUntilQuiescence();
+        tester.completeUpgrade(default0, version2, "default");
+    }
+
     private static void writeControllerVersion(HostName hostname, Version version, CuratorDb db) {
         db.writeControllerVersion(hostname, new ControllerVersion(version, "badc0ffee", Instant.EPOCH));
     }