Take care of DeploymentTester stragglers

8 files changed, 68 insertions, 107 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java
index d1303c4a7ed..63d84926144 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/BuildJob.java
@@ -17,13 +17,6 @@ import java.util.function.Consumer;
 
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component;
 
-/**
- * Create a build job for testing purposes. In most cases this should be constructed by calling
- * {@link DeploymentTester#jobCompletion(JobType)} or {@link ContainerControllerTester#jobCompletion(JobType)}.
- *
- * @author mpolden
- */
-
 public class BuildJob {
 
     public static final long defaultBuildNumber = 42;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 09d93a5f857..38c9b6e3e49 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -104,12 +104,13 @@ public class DeploymentTriggerTest {
         assertEquals("Job is not triggered when no projectId is present", 0, tester.jobs().active().size());
     }
 
+    /*
     @Test
     @Ignore
     // TODO jonmv: Re-enable, but changed, when instances are orchestrated.
     public void testIndependentInstances() {
-        Application app1 = tester.tester().createApplication("instance1", "app", "tenant", 1, 1L);
-        Application app2 = tester.tester().createApplication("instance2", "app", "tenant", 2, 1L);
+        var app1 = tester.tester().createApplication("instance1", "app", "tenant", 1, 1L);
+        var app2 = tester.tester().createApplication("instance2", "app", "tenant", 2, 1L);
         Instance instance1 = tester.tester().instance(app1.id().instance(InstanceName.from("instance1")));
         Instance instance2 = tester.tester().instance(app2.id().instance(InstanceName.from("instance2")));
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
@@ -147,6 +148,7 @@ public class DeploymentTriggerTest {
         assertEquals(newVersion, instance1Version);
         assertEquals(version, instance2Version);
     }
+    */
 
     @Test
     public void abortsJobsOnNewApplicationChange() {
@@ -201,11 +203,11 @@ public class DeploymentTriggerTest {
 
         // Test jobs pass
         app.runJob(systemTest).runJob(stagingTest);
-        tester.tester().deploymentTrigger().triggerReadyJobs();
+        tester.triggerJobs();
 
         // No jobs have started yet, as 30 seconds have not yet passed.
         assertEquals(0, tester.jobs().active().size());
-        tester.tester().clock().advance(Duration.ofSeconds(30));
+        tester.clock().advance(Duration.ofSeconds(30));
         tester.triggerJobs();
 
         // 30 seconds later, the first jobs may trigger.
@@ -213,7 +215,7 @@ public class DeploymentTriggerTest {
         app.assertRunning(productionUsWest1);
 
         // 3 minutes pass, delayed trigger does nothing as us-west-1 is still in progress
-        tester.tester().clock().advance(Duration.ofMinutes(3));
+        tester.clock().advance(Duration.ofMinutes(3));
         tester.triggerJobs();
         assertEquals(1, tester.jobs().active().size());
         app.assertRunning(productionUsWest1);
@@ -226,18 +228,18 @@ public class DeploymentTriggerTest {
         assertTrue("No more jobs triggered at this time", tester.jobs().active().isEmpty());
 
         // 3 minutes pass, us-central-1 is still not triggered
-        tester.tester().clock().advance(Duration.ofMinutes(3));
+        tester.clock().advance(Duration.ofMinutes(3));
         tester.triggerJobs();
         assertTrue("No more jobs triggered at this time", tester.jobs().active().isEmpty());
 
         // 4 minutes pass, us-central-1 is triggered
-        tester.tester().clock().advance(Duration.ofMinutes(1));
+        tester.clock().advance(Duration.ofMinutes(1));
         tester.triggerJobs();
         app.runJob(productionUsCentral1);
         assertTrue("All jobs consumed", tester.jobs().active().isEmpty());
 
         // Delayed trigger job runs again, with nothing to trigger
-        tester.tester().clock().advance(Duration.ofMinutes(10));
+        tester.clock().advance(Duration.ofMinutes(10));
         tester.triggerJobs();
         assertTrue("All jobs consumed", tester.jobs().active().isEmpty());
     }
@@ -423,15 +425,15 @@ public class DeploymentTriggerTest {
         app.assertNotRunning(productionUsWest1);
 
         // us-west-1 triggers when no longer paused, but does not retry when paused again.
-        tester.tester().clock().advance(Duration.ofMillis(1500));
+        tester.clock().advance(Duration.ofMillis(1500));
         tester.triggerJobs();
         app.assertRunning(productionUsWest1);
-        tester.deploymentTrigger().pauseJob(app.instanceId(), productionUsWest1, tester.tester().clock().instant().plus(Duration.ofSeconds(1)));
+        tester.deploymentTrigger().pauseJob(app.instanceId(), productionUsWest1, tester.clock().instant().plus(Duration.ofSeconds(1)));
         app.failDeployment(productionUsWest1);
         tester.triggerJobs();
         app.assertNotRunning(productionUsWest1);
 
-        tester.tester().clock().advance(Duration.ofMillis(1000));
+        tester.clock().advance(Duration.ofMillis(1000));
         tester.triggerJobs();
         app.runJob(productionUsWest1);
 
@@ -466,7 +468,7 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(appVersion1), app.application().change());
 
         // Now cancel the change as is done through the web API.
-        tester.tester().deploymentTrigger().cancelChange(app.application().id(), ALL);
+        tester.deploymentTrigger().cancelChange(app.application().id(), ALL);
         assertEquals(Change.empty(), app.application().change());
 
         // A new version is released, which should now deploy the currently deployed application version to avoid downgrades.
@@ -681,13 +683,13 @@ public class DeploymentTriggerTest {
                      initialFailure, app.instance().deploymentJobs().jobStatus().get(systemTest).firstFailing().get().at());
 
         // Failure again -- failingSince should remain the same
-        tester.tester().clock().advance(Duration.ofMillis(1000));
+        tester.clock().advance(Duration.ofMillis(1000));
         app.failDeployment(systemTest);
         assertEquals("Failure age is right at second consecutive failure",
                      initialFailure, app.instance().deploymentJobs().jobStatus().get(systemTest).firstFailing().get().at());
 
         // Success resets failingSince
-        tester.tester().clock().advance(Duration.ofMillis(1000));
+        tester.clock().advance(Duration.ofMillis(1000));
         app.runJob(systemTest);
         assertFalse(app.instance().deploymentJobs().jobStatus().get(systemTest).firstFailing().isPresent());
 
@@ -696,8 +698,8 @@ public class DeploymentTriggerTest {
 
         // Two repeated failures again.
         // Initial failure
-        tester.tester().clock().advance(Duration.ofMillis(1000));
-        initialFailure = tester.tester().clock().instant().truncatedTo(MILLIS);
+        tester.clock().advance(Duration.ofMillis(1000));
+        initialFailure = tester.clock().instant().truncatedTo(MILLIS);
 
         app.submit(applicationPackage);
         app.failDeployment(systemTest);
@@ -705,7 +707,7 @@ public class DeploymentTriggerTest {
                      initialFailure, app.instance().deploymentJobs().jobStatus().get(systemTest).firstFailing().get().at());
 
         // Failure again -- failingSince should remain the same
-        tester.tester().clock().advance(Duration.ofMillis(1000));
+        tester.clock().advance(Duration.ofMillis(1000));
         app.failDeployment(systemTest);
         assertEquals("Failure age is right at second consecutive failure",
                      initialFailure, app.instance().deploymentJobs().jobStatus().get(systemTest).firstFailing().get().at());
@@ -767,10 +769,10 @@ public class DeploymentTriggerTest {
 
         // all applications: system-test completes successfully with some time in between, to determine trigger order.
         app2.runJob(systemTest);
-        tester.tester().clock().advance(Duration.ofMinutes(1));
+        tester.clock().advance(Duration.ofMinutes(1));
 
         app1.runJob(systemTest);
-        tester.tester().clock().advance(Duration.ofMinutes(1));
+        tester.clock().advance(Duration.ofMinutes(1));
 
         app3.runJob(systemTest);
 
@@ -879,8 +881,8 @@ public class DeploymentTriggerTest {
                 .build();
         var app = tester.deploymentContext().submit(applicationPackage); // TODO jonmv: support instances in deployment context>
         app.deploy();
-        assertEquals(2, tester.tester().application(appId).instances().size());
-        assertEquals(2, tester.tester().application(appId).productionDeployments().values().stream()
+        assertEquals(2, app.application().instances().size());
+        assertEquals(2, app.application().productionDeployments().values().stream()
                               .mapToInt(Collection::size)
                               .sum());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
index fc38d9d1f52..d257392298e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java
@@ -61,44 +61,46 @@ public class InternalDeploymentTester {
     public static final String athenzDomain = "domain";
 
     private final DeploymentContext defaultContext;
-    private final DeploymentTester tester;
+    private final ControllerTester tester;
     private final JobController jobs;
     private final RoutingGeneratorMock routing;
     private final MockTesterCloud cloud;
     private final JobRunner runner;
+    private final Upgrader upgrader;
     private final ReadyJobsTrigger readyJobsTrigger;
     private final OutstandingChangeDeployer outstandingChangeDeployer;
     private final NameServiceDispatcher nameServiceDispatcher;
 
-    public DeploymentTester tester() { return tester; }
     public JobController jobs() { return jobs; }
     public RoutingGeneratorMock routing() { return routing; }
     public MockTesterCloud cloud() { return cloud; }
     public JobRunner runner() { return runner; }
     public ConfigServerMock configServer() { return tester.configServer(); }
     public Controller controller() { return tester.controller(); }
-    public DeploymentTrigger deploymentTrigger() { return tester.deploymentTrigger(); }
-    public ControllerTester controllerTester() { return tester.controllerTester(); }
-    public Upgrader upgrader() { return tester.upgrader(); }
-    public ApplicationController applications() { return tester.applications(); }
+    public DeploymentTrigger deploymentTrigger() { return applications().deploymentTrigger(); }
+    public ControllerTester controllerTester() { return tester; }
+    public Upgrader upgrader() { return upgrader; }
+    public ApplicationController applications() { return tester.controller().applications(); }
     public ManualClock clock() { return tester.clock(); }
-    public Application application() { return tester.application(appId); }
-    public Application application(TenantAndApplicationId id ) { return tester.application(id); }
-    public Instance instance() { return tester.instance(instanceId); }
-    public Instance instance(ApplicationId id) { return tester.instance(id); }
+    public Application application() { return application(appId); }
+    public Application application(TenantAndApplicationId id ) { return applications().requireApplication(id); }
+    public Instance instance() { return instance(instanceId); }
+    public Instance instance(ApplicationId id) { return applications().requireInstance(id); }
 
     public InternalDeploymentTester() {
         this(new ControllerTester());
     }
 
     public InternalDeploymentTester(ControllerTester controllerTester) {
-        tester = new DeploymentTester(controllerTester);
+        tester = controllerTester;
         jobs = tester.controller().jobController();
-        routing = tester.controllerTester().serviceRegistry().routingGeneratorMock();
+        routing = tester.serviceRegistry().routingGeneratorMock();
         cloud = (MockTesterCloud) tester.controller().jobController().cloud();
         var jobControl = new JobControl(tester.controller().curator());
         runner = new JobRunner(tester.controller(), Duration.ofDays(1), jobControl,
                                JobRunnerTest.inThreadExecutor(), new InternalStepRunner(tester.controller()));
+        upgrader = new Upgrader(tester.controller(), maintenanceInterval, jobControl, tester.curator());
+        upgrader.setUpgradesPerMinute(1); // Anything that makes it at least one for any maintenance period is fine.
         readyJobsTrigger = new ReadyJobsTrigger(tester.controller(), maintenanceInterval, jobControl);
         outstandingChangeDeployer = new OutstandingChangeDeployer(tester.controller(), maintenanceInterval, jobControl);
         nameServiceDispatcher = new NameServiceDispatcher(tester.controller(), maintenanceInterval, jobControl,
@@ -109,10 +111,10 @@ public class InternalDeploymentTester {
         // Get deployment job logs to stderr.
         Logger.getLogger("").setLevel(LogLevel.DEBUG);
         Logger.getLogger(InternalStepRunner.class.getName()).setLevel(LogLevel.DEBUG);
-        tester.controllerTester().configureDefaultLogHandler(handler -> handler.setLevel(LogLevel.DEBUG));
+        tester.configureDefaultLogHandler(handler -> handler.setLevel(LogLevel.DEBUG));
 
         // Mock Athenz domain to allow launch of service
-        AthenzDbMock.Domain domain = tester.controllerTester().athenzDb().getOrCreateDomain(new com.yahoo.vespa.athenz.api.AthenzDomain(ATHENZ_DOMAIN));
+        AthenzDbMock.Domain domain = tester.athenzDb().getOrCreateDomain(new com.yahoo.vespa.athenz.api.AthenzDomain(ATHENZ_DOMAIN));
         domain.services.put(ATHENZ_SERVICE, new AthenzDbMock.Service(true));
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index fc73af6acf7..57ec2d3130d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -78,44 +78,12 @@ public class InternalStepRunnerTest {
     }
 
     private SystemName system() {
-        return tester.tester().controller().system();
+        return tester.controller().system();
     }
 
     @Test
     public void canRegisterAndRunDirectly() {
-        tester.deployNewSubmission(tester.newSubmission());
-
-        Version version = new Version("7.1");
-        tester.tester().upgradeSystem(version);
-        tester.deployNewPlatform(version);
-    }
-
-    @Test
-    public void canSwitchFromScrewdriverAndBackAgain() {
-        // Deploys a default application package with default build number.
-        tester.tester().deployCompletely(tester.application(), DeploymentContext.applicationPackage);
-        tester.setEndpoints(instanceId, JobType.productionUsCentral1.zone(system()));
-        tester.setEndpoints(instanceId, JobType.productionUsWest1.zone(system()));
-        tester.setEndpoints(instanceId, JobType.productionUsEast3.zone(system()));
-
-        // Let application have an ongoing upgrade when it switches (but kill the jobs, as the tester assumes they aren't running).
-        tester.tester().upgradeSystem(new Version("7.1"));
-        tester.tester().buildService().clear();
-
-        tester.deployNewSubmission(tester.newSubmission());
-        tester.deployNewSubmission(tester.newSubmission());
-
-        Version version = new Version("7.2");
-        tester.tester().upgradeSystem(version);
-        tester.deployNewPlatform(version);
-
-        tester.jobs().unregister(appId);
-        try {
-            tester.tester().deployCompletely(tester.application(), DeploymentContext.applicationPackage, BuildJob.defaultBuildNumber + 1);
-            throw new IllegalStateException("Component job should get even again with build numbers to produce a change.");
-        }
-        catch (AssertionError expected) { }
-        tester.tester().deployCompletely(tester.application(), DeploymentContext.applicationPackage, BuildJob.defaultBuildNumber + 2);
+        tester.deploymentContext().submit().deploy();
     }
 
     @Test
@@ -247,13 +215,13 @@ public class InternalStepRunnerTest {
         assertEquals(unfinished, tester.jobs().last(instanceId, JobType.systemTest).get().steps().get(Step.installReal));
         assertEquals(unfinished, tester.jobs().last(instanceId, JobType.systemTest).get().steps().get(Step.installTester));
 
-        tester.tester().controller().curator().writeRoutingPolicies(instanceId, Set.of(new RoutingPolicy(instanceId,
+        tester.controller().curator().writeRoutingPolicies(instanceId, Set.of(new RoutingPolicy(instanceId,
                                                                                                          ClusterSpec.Id.from("default"),
                                                                                                          JobType.systemTest.zone(system()),
                                                                                                          HostName.from("host"),
                                                                                                          Optional.empty(),
                                                                                                          emptySet())));
-        tester.tester().controller().curator().writeRoutingPolicies(testerId.id(), Set.of(new RoutingPolicy(testerId.id(),
+        tester.controller().curator().writeRoutingPolicies(testerId.id(), Set.of(new RoutingPolicy(testerId.id(),
                                                                                                             ClusterSpec.Id.from("default"),
                                                                                                             JobType.systemTest.zone(system()),
                                                                                                             HostName.from("host"),
@@ -367,7 +335,7 @@ public class InternalStepRunnerTest {
         tester.configServer().setVersion(instanceId, zone, version);
         tester.runner().run();
         assertEquals(1, tester.jobs().active().size());
-        assertEquals(version, tester.tester().instance(instanceId).deployments().get(zone).version());
+        assertEquals(version, tester.instance(instanceId).deployments().get(zone).version());
 
         try {
             tester.jobs().deploy(instanceId, JobType.productionApNortheast1, Optional.empty(), applicationPackage);
@@ -381,7 +349,7 @@ public class InternalStepRunnerTest {
         tester.startSystemTestTests();
         tester.cloud().set(TesterCloud.Status.NOT_STARTED);
         tester.runner().run();
-        MockMailer mailer = ((MockMailer) tester.tester().controller().serviceRegistry().mailer());
+        MockMailer mailer = ((MockMailer) tester.controller().serviceRegistry().mailer());
         assertEquals(1, mailer.inbox("a@b").size());
         assertEquals("Vespa application tenant.application: System test failing due to system error",
                      mailer.inbox("a@b").get(0).subject());
@@ -415,8 +383,8 @@ public class InternalStepRunnerTest {
 
     @Test
     public void certificateTimeoutAbortsJob() {
-        tester.tester().controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd);
-        tester.tester().controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.aws-us-east-1c"));
+        tester.controllerTester().zoneRegistry().setSystemName(SystemName.PublicCd);
+        tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.aws-us-east-1c"));
         RunId id = tester.startSystemTestTests();
 
         List<X509Certificate> trusted = new ArrayList<>(publicCdApplicationPackage.trustedCertificates());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
index e88452bd54e..d0ba13cf5bc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainerTest.java
@@ -3,18 +3,15 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.ClusterMetrics;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester;
 import org.junit.Test;
 
@@ -22,7 +19,6 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Optional;
 import java.util.function.Supplier;
 
 import static java.time.temporal.ChronoUnit.MILLIS;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index d35d6d1d4a0..c2cb427aaab 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -191,7 +191,7 @@ public class MetricsReporterTest {
                             .deploy();
         assertEquals(1000, context.lastSubmission().get().buildTime().get().toEpochMilli());
 
-        MetricsReporter reporter = createReporter(tester.tester().controller());
+        MetricsReporter reporter = createReporter(tester.controller());
         reporter.maintain();
         assertEquals(tester.clock().instant().getEpochSecond() - 1,
                      getMetric(MetricsReporter.DEPLOYMENT_BUILD_AGE_SECONDS, context.instanceId()));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 1105a478e14..a60c195519a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -65,14 +65,14 @@ public class JobControllerApiHandlerHelperTest {
         tester.runJob(stagingTest);
         tester.runJob(productionUsCentral1);
 
-        tester.tester().readyJobTrigger().maintain();
+        tester.triggerJobs();
 
         // us-east-3 eats the deployment failure and fails before deployment, while us-west-1 fails after.
         tester.configServer().throwOnNextPrepare(new ConfigServerException(URI.create("url"), "ERROR!", INVALID_APPLICATION_PACKAGE, null));
         tester.runner().run();
         assertEquals(deploymentFailed, tester.jobs().last(instanceId, productionUsEast3).get().status());
 
-        ZoneId usWest1 = productionUsWest1.zone(tester.tester().controller().system());
+        ZoneId usWest1 = productionUsWest1.zone(tester.controller().system());
         tester.configServer().convergeServices(instanceId, usWest1);
         tester.configServer().convergeServices(testerId.id(), usWest1);
         tester.setEndpoints(instanceId, usWest1);
@@ -81,9 +81,9 @@ public class JobControllerApiHandlerHelperTest {
         tester.cloud().set(FAILURE);
         tester.runner().run();
         assertEquals(testFailure, tester.jobs().last(instanceId, productionUsWest1).get().status());
-        assertEquals(revision2, tester.instance().deployments().get(productionUsCentral1.zone(tester.tester().controller().system())).applicationVersion());
-        assertEquals(revision1, tester.instance().deployments().get(productionUsEast3.zone(tester.tester().controller().system())).applicationVersion());
-        assertEquals(revision2, tester.instance().deployments().get(productionUsWest1.zone(tester.tester().controller().system())).applicationVersion());
+        assertEquals(revision2, tester.instance().deployments().get(productionUsCentral1.zone(tester.controller().system())).applicationVersion());
+        assertEquals(revision1, tester.instance().deployments().get(productionUsEast3.zone(tester.controller().system())).applicationVersion());
+        assertEquals(revision2, tester.instance().deployments().get(productionUsWest1.zone(tester.controller().system())).applicationVersion());
 
         tester.clock().advance(Duration.ofMillis(1000));
 
@@ -91,24 +91,26 @@ public class JobControllerApiHandlerHelperTest {
         tester.newSubmission();
         tester.runJob(systemTest);
         tester.runJob(stagingTest);
-        tester.tester().readyJobTrigger().maintain(); // Starts a run for us-central-1.
-        tester.tester().readyJobTrigger().maintain(); // Starts a new staging test run.
+        tester.triggerJobs(); // Starts a run for us-central-1.
+        tester.triggerJobs(); // Starts a new staging test run.
         tester.runner().run();
         assertEquals(running, tester.jobs().last(instanceId, productionUsCentral1).get().status());
         assertEquals(running, tester.jobs().last(instanceId, stagingTest).get().status());
 
         // Staging is expired, and the job fails and won't be retried immediately.
-        tester.tester().controller().applications().deactivate(instanceId, stagingTest.zone(tester.tester().controller().system()));
+        tester.controller().applications().deactivate(instanceId, stagingTest.zone(tester.controller().system()));
         tester.runner().run();
         assertEquals(installationFailed, tester.jobs().last(instanceId, stagingTest).get().status());
 
         tester.clock().advance(Duration.ofMillis(100_000)); // More than the minute within which there are immediate retries.
-        tester.tester().readyJobTrigger().maintain();
+        tester.triggerJobs();
         assertEquals(installationFailed, tester.jobs().last(instanceId, stagingTest).get().status());
 
         // System upgrades to a new version, which won't yet start.
         Version platform = new Version("7.1");
-        tester.tester().upgradeSystem(platform);
+        tester.controllerTester().upgradeSystem(platform);
+        tester.upgrader().maintain();
+        tester.triggerJobs();
 
         // us-central-1 has started, deployed, and is installing. Deployment is not yet verified.
         // us-east-3 is waiting for the failed staging test and us-central-1, while us-west-1 is waiting only for us-central-1.
@@ -117,7 +119,7 @@ public class JobControllerApiHandlerHelperTest {
         // one success from revision2 to revision3 and one failure from revision1 to revision3.
         assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(instanceId, stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json");
         assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(instanceId, productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json");
-        assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.tester().controller(), instanceId, URI.create("https://some.url:43/root/")), "overview.json");
+        assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), instanceId, URI.create("https://some.url:43/root/")), "overview.json");
 
         tester.runJob(instanceId, JobType.devAwsUsEast2a, applicationPackage);
         assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(instanceId, devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json");
@@ -128,7 +130,7 @@ public class JobControllerApiHandlerHelperTest {
         InternalDeploymentTester tester = new InternalDeploymentTester();
         tester.clock().setInstant(Instant.EPOCH);
 
-        ZoneId zone = JobType.devUsEast1.zone(tester.tester().controller().system());
+        ZoneId zone = JobType.devUsEast1.zone(tester.controller().system());
         tester.jobs().deploy(instanceId, JobType.devUsEast1, Optional.empty(), applicationPackage);
         tester.configServer().setLogStream("1554970337.935104\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n");
         assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(instanceId, devUsEast1).get().id(), null), "dev-us-east-1-log-first-part.json");
@@ -138,7 +140,7 @@ public class JobControllerApiHandlerHelperTest {
         tester.setEndpoints(instanceId, zone);
         tester.runner().run();
 
-        assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.tester().controller(), instanceId, URI.create("https://some.url:43/root")), "dev-overview.json");
+        assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), instanceId, URI.create("https://some.url:43/root")), "dev-overview.json");
         assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(instanceId, devUsEast1).get().id(), "9"), "dev-us-east-1-log-second-part.json");
     }
 
@@ -149,11 +151,11 @@ public class JobControllerApiHandlerHelperTest {
         var region = "us-west-1";
         var applicationPackage = new ApplicationPackageBuilder().region(region).build();
         // Deploy directly to production zone, like integration tests.
-        tester.tester().controller().applications().deploy(tester.instance().id(), ZoneId.from("prod", region),
+        tester.controller().applications().deploy(tester.instance().id(), ZoneId.from("prod", region),
                                                            Optional.of(applicationPackage),
                                                            new DeployOptions(true, Optional.empty(),
                                                                              false, false));
-        assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.tester().controller(), instanceId, URI.create("https://some.url:43/root/")),
+        assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), instanceId, URI.create("https://some.url:43/root/")),
                        "jobs-direct-deployment.json");
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
index 899a0228b48..bd4fd3d29f7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java
@@ -177,11 +177,10 @@ public class VersionStatusTest {
     
     @Test
     public void testVersionConfidence() {
-        InternalDeploymentTester tester = new InternalDeploymentTester();
+        InternalDeploymentTester tester = new InternalDeploymentTester().atHourOfDay(5);
         Version version0 = new Version("6.2");
         tester.controllerTester().upgradeSystem(version0);
         tester.upgrader().maintain();
-
         var builder = new ApplicationPackageBuilder().region("us-west-1").region("us-east-3");
 
         // Setup applications - all running on version0
@@ -421,9 +420,8 @@ public class VersionStatusTest {
 
     @Test
     public void testConfidenceChangeRespectsTimeWindow() {
-        InternalDeploymentTester tester = new InternalDeploymentTester();
+        InternalDeploymentTester tester = new InternalDeploymentTester().atHourOfDay(5);
         // Canaries and normal application deploys on initial version
-        assertEquals(5, tester.controllerTester().hourOfDayAfter(Duration.ZERO));
         Version version0 = Version.fromString("7.1");
         tester.controllerTester().upgradeSystem(version0);
         var canary0 = tester.newDeploymentContext("tenant1", "canary0", "default")