Merge pull request #6514 from vespa-engine/jvenstad/deployments

Jvenstad/deployments

16 files changed, 54 insertions, 124 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
index e91a5909f80..56c2ee8da6b 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/BuildService.java
@@ -20,11 +20,6 @@ public interface BuildService {
      */
     JobState stateOf(BuildJob buildJob);
 
-    /**
-     * Returns whether the given build job should be performed by this build service.
-     */
-    default boolean builds(BuildJob buildJob) { return true; }
-
     enum JobState {
 
         /** Job is not running, and may be triggered. */
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/Testers.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
index dccc0e47ceb..f1d07fc9097 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/Testers.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/TesterCloud.java
@@ -7,7 +7,7 @@ import java.net.URI;
  *
  * @author jonmv
  */
-public interface Testers {
+public interface TesterCloud {
 
     /** Signals the tester to run its tests. */
     void startTests(URI testerUrl, Suite suite, byte[] config);
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesters.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
index 021e4d7f293..c2199c284f3 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesters.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockTesterCloud.java
@@ -1,16 +1,14 @@
 package com.yahoo.vespa.hosted.controller.api.integration.stubs;
 
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 
 import java.net.URI;
 import java.util.Arrays;
 
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.FAILURE;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.NOT_STARTED;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.RUNNING;
-import static com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers.Status.SUCCESS;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.NOT_STARTED;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.RUNNING;
 
-public class MockTesters implements Testers {
+public class MockTesterCloud implements TesterCloud {
 
     private byte[] logs = new byte[0];
     private Status status = NOT_STARTED;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 790d6d00035..f8be9f55b84 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -26,8 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.GlobalRoutingSe
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RotationStatus;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
-import com.yahoo.vespa.hosted.controller.deployment.DelegatingBuildService;
-import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.rotation.Rotation;
@@ -126,8 +124,7 @@ public class Controller extends AbstractComponent {
                                                           configServer,
                                                           Objects.requireNonNull(artifactRepository, "ArtifactRepository cannot be null"),
                                                           Objects.requireNonNull(routingGenerator, "RoutingGenerator cannot be null"),
-                                                          new DelegatingBuildService(Objects.requireNonNull(buildService, "BuildService cannot be null"),
-                                                                                     new InternalBuildService(jobController)),
+                                                          Objects.requireNonNull(buildService, "BuildService cannot be null"),
                                                           clock);
         tenantController = new TenantController(this, curator, athenzClientFactory);
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java
deleted file mode 100644
index d2159841c9d..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DelegatingBuildService.java
+++ /dev/null
@@ -1,30 +0,0 @@
-package com.yahoo.vespa.hosted.controller.deployment;
-
-import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
-
-/**
- * Sends build jobs to an internal build system whenever it accepts them, or to an external one otherwise.
- *
- * @author jonmv
- */
-public class DelegatingBuildService implements BuildService {
-
-    private final BuildService external;
-    private final BuildService internal;
-
-    public DelegatingBuildService(BuildService external, BuildService internal) {
-        this.external = external;
-        this.internal = internal;
-    }
-
-    @Override
-    public void trigger(BuildJob buildJob) {
-        (internal.builds(buildJob) ? internal : external).trigger(buildJob);
-    }
-
-    @Override
-    public JobState stateOf(BuildJob buildJob) {
-        return (internal.builds(buildJob) ? internal : external).stateOf(buildJob);
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index e3b4b4cef8c..3bfce2725a2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -69,11 +69,13 @@ public class DeploymentTrigger {
     private final Controller controller;
     private final Clock clock;
     private final BuildService buildService;
+    private final JobController jobs;
 
     public DeploymentTrigger(Controller controller, BuildService buildService, Clock clock) {
         this.controller = Objects.requireNonNull(controller, "controller cannot be null");
-        this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null");
         this.clock = Objects.requireNonNull(clock, "clock cannot be null");
+        this.buildService = Objects.requireNonNull(buildService, "buildService cannot be null");
+        this.jobs = controller.jobController();
     }
 
     public DeploymentSteps steps(DeploymentSpec spec) {
@@ -161,14 +163,19 @@ public class DeploymentTrigger {
      * Attempts to trigger the given job for the given application and returns the outcome.
      *
      * If the build service can not find the given job, or claims it is illegal to trigger it,
-     * the project id is removed from the application owning the job, to prevent further trigger attemps.
+     * the project id is removed from the application owning the job, to prevent further trigger attempts.
      */
     public boolean trigger(Job job) {
         log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering));
         try {
-            buildService.trigger(job);
-            applications().lockOrThrow(job.applicationId(), application ->
-                    applications().store(application.withJobTriggering(job.jobType, job.triggering)));
+            applications().lockOrThrow(job.applicationId(), application -> {
+                if (application.get().deploymentJobs().builtInternally())
+                    jobs.start(job.applicationId(), job.jobType);
+                else
+                    buildService.trigger(job);
+
+                applications().store(application.withJobTriggering(job.jobType, job.triggering));
+            });
             return true;
         }
         catch (RuntimeException e) {
@@ -184,6 +191,9 @@ public class DeploymentTrigger {
     public List<JobType> forceTrigger(ApplicationId applicationId, JobType jobType, String user) {
         Application application = applications().require(applicationId);
         if (jobType == component) {
+            if (application.deploymentJobs().builtInternally())
+                throw new IllegalArgumentException(applicationId + " has no component job we can trigger.");
+
             buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName()));
             return singletonList(component);
         }
@@ -339,6 +349,7 @@ public class DeploymentTrigger {
         if (!jobStatus.get().lastCompleted().isPresent()) return true; // Never completed
         if (!jobStatus.get().firstFailing().isPresent()) return true; // Should not happen as firstFailing should be set for an unsuccessful job
         if (!versions.targetsMatch(jobStatus.get().lastCompleted().get())) return true; // Always trigger as targets have changed
+        if (application.deploymentSpec().upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return true; // Don't throttle canaries
 
         Instant firstFailing = jobStatus.get().firstFailing().get().at();
         Instant lastCompleted = jobStatus.get().lastCompleted().get().at();
@@ -376,6 +387,10 @@ public class DeploymentTrigger {
     }
 
     private JobState jobStateOf(Application application, JobType jobType) {
+        if (application.deploymentJobs().builtInternally()) {
+            Optional<RunStatus> run = controller.jobController().last(application.id(), jobType);
+            return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle;
+        }
         return buildService.stateOf(BuildJob.of(application.id(),
                                                 application.deploymentJobs().projectId().getAsLong(),
                                                 jobType.jobName()));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java
deleted file mode 100644
index 18b62f5ea0f..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalBuildService.java
+++ /dev/null
@@ -1,39 +0,0 @@
-package com.yahoo.vespa.hosted.controller.deployment;
-
-import com.yahoo.vespa.hosted.controller.api.integration.BuildService;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
-
-import java.util.Optional;
-
-/**
- * Wraps a JobController as a BuildService.
- *
- * Shall be inlined when the {@link DelegatingBuildService} delegates all jobs to it.
- *
- * @author jonmv
- */
-public class InternalBuildService implements BuildService {
-
-    private final JobController jobs;
-
-    public InternalBuildService(JobController jobs) {
-        this.jobs = jobs;
-    }
-
-    @Override
-    public void trigger(BuildJob buildJob) {
-        jobs.start(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName()));
-    }
-
-    @Override
-    public JobState stateOf(BuildJob buildJob) {
-        Optional<RunStatus> run = jobs.last(buildJob.applicationId(), JobType.fromJobName(buildJob.jobName()));
-        return run.isPresent() && ! run.get().hasEnded() ? JobState.running : JobState.idle;
-    }
-
-    @Override
-    public boolean builds(BuildJob buildJob) {
-        return jobs.builds(buildJob.applicationId());
-    }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index c5bcffd7ffe..5c848ba279b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -20,7 +20,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareRes
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.ApplicationVersion;
@@ -84,12 +84,12 @@ public class InternalStepRunner implements StepRunner {
     }
 
     private final Controller controller;
-    private final Testers testers;
+    private final TesterCloud testerCloud;
     private final ThreadLocal<ByteArrayLogger> logger = new ThreadLocal<>();
 
-    public InternalStepRunner(Controller controller, Testers testers) {
+    public InternalStepRunner(Controller controller, TesterCloud testerCloud) {
         this.controller = controller;
-        this.testers = testers;
+        this.testerCloud = testerCloud;
     }
 
     @Override
@@ -301,9 +301,9 @@ public class InternalStepRunner implements StepRunner {
         Optional<URI> testerEndpoint = testerEndpoint(id);
         if (testerEndpoint.isPresent()) {
             logger.get().log("Starting tests ...");
-            testers.startTests(testerEndpoint.get(),
-                               Testers.Suite.of(id.type()),
-                               testConfig(id.application(), zone(id.type()), controller.system(), endpoints));
+            testerCloud.startTests(testerEndpoint.get(),
+                                   TesterCloud.Suite.of(id.type()),
+                                   testConfig(id.application(), zone(id.type()), controller.system(), endpoints));
             return succeeded;
         }
 
@@ -321,7 +321,7 @@ public class InternalStepRunner implements StepRunner {
                 .orElseThrow(() -> new NoSuchElementException("Endpoint for tester vanished again before tests were complete!"));
 
         Status status;
-        switch (testers.getStatus(testerEndpoint)) {
+        switch (testerCloud.getStatus(testerEndpoint)) {
             case NOT_STARTED:
                 throw new IllegalStateException("Tester reports tests not started, even though they should have!");
             case RUNNING:
@@ -339,7 +339,7 @@ public class InternalStepRunner implements StepRunner {
             default:
                 throw new AssertionError("Unknown status!");
         }
-        logger.get().log(new String(testers.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless!
+        logger.get().log(new String(testerCloud.getLogs(testerEndpoint))); // TODO jvenstad: Replace with something less hopeless!
         return status;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
index aaf43097908..c4aac48503f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunResult.java
@@ -1,7 +1,7 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 /**
- * Outcomes of jobs run by an {@link InternalBuildService}.
+ * Outcomes of jobs run by a {@link JobController}.
  *
  * @author jonmv
  */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
index 1fd32524c88..c6724255a58 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/RunStatus.java
@@ -16,7 +16,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinishe
 import static java.util.Objects.requireNonNull;
 
 /**
- * Immutable class containing status information for a deployment job run by an {@link InternalBuildService}.
+ * Immutable class containing status information for a deployment job run by a {@link JobController}.
  *
  * @author jonmv
  */
@@ -83,7 +83,7 @@ public class RunStatus {
     public Optional<RunResult> result() {
 
         // No result of not finished yet
-        if (!hasEnded()) return Optional.empty();
+        if ( ! hasEnded()) return Optional.empty();
 
         // If any steps has failed - then we need to figure out what - for now return fixed error result
         if (hasFailed()) return Optional.of(RunResult.testError);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
index e1e2281c5ea..f734dde9440 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java
@@ -66,10 +66,6 @@ public enum Step {
 
     public List<Step> prerequisites() { return prerequisites; }
 
-    public static Step last() {
-        return report;
-    }
-
     public enum Status {
 
         /** Step still has unsatisfied finish criteria -- it may not even have started. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 7fa16a02649..e663b0154d7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -4,13 +4,12 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.jdisc.Metric;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.integration.deployment.Testers;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues;
 import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
-import com.yahoo.vespa.hosted.controller.deployment.DummyStepRunner;
 import com.yahoo.vespa.hosted.controller.deployment.InternalStepRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
@@ -45,7 +44,7 @@ public class ControllerMaintenance extends AbstractComponent {
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
     public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator,
-                                 JobControl jobControl, Metric metric, Chef chefClient, Testers testers,
+                                 JobControl jobControl, Metric metric, Chef chefClient, TesterCloud testerCloud,
                                  DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues,
                                  NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient) {
         Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
@@ -63,7 +62,7 @@ public class ControllerMaintenance extends AbstractComponent {
         applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues);
         dnsMaintainer = new DnsMaintainer(controller, Duration.ofHours(12), jobControl, nameService);
         systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl);
-        jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testers));
+        jobRunner = new JobRunner(controller, Duration.ofSeconds(30), jobControl, new InternalStepRunner(controller, testerCloud));
     }
 
     public Upgrader upgrader() { return upgrader; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index 7dbf1a2c05e..9fa67cb5f89 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.deployment.InternalBuildService;
 import com.yahoo.vespa.hosted.controller.deployment.JobController;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
 import com.yahoo.vespa.hosted.controller.deployment.RunStatus;
@@ -20,9 +19,8 @@ import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.logging.Logger;
 
 /**
- * Advances the set of {@link RunStatus}es for an {@link InternalBuildService}.
+ * Advances the set of {@link RunStatus}es for a {@link JobController}.
  *
- * @see JobController
  * @author jonmv
  */
 public class JobRunner extends Maintainer {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index af7261149ad..95eb9117c5a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -480,7 +480,6 @@ public class DeploymentTriggerTest {
                 .environment(Environment.prod)
                 .region("us-central-1")
                 .region("eu-west-1")
-                .upgradePolicy("canary")
                 .build();
         tester.deployCompletely(application, applicationPackage);
 
@@ -497,7 +496,7 @@ public class DeploymentTriggerTest {
         tester.clock().advance(Duration.ofHours(1));
 
         Version v1 = new Version("7.1");
-        tester.upgradeSystem(v1); // Downgrade, but it works, since the app is a canary.
+        tester.upgradeSystem(v1); // Downgrade to cut down on the amount of code.
         assertEquals(Change.of(v1), app.get().change());
 
         // 7.1 proceeds 'til the last job, where it fails; us-central-1 is skipped, as current change is strictly dominated by what's deployed there.
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index de645cff96c..639cfad9958 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -81,7 +81,7 @@ public class ControllerContainerTest {
             "  <component id='com.yahoo.vespa.hosted.controller.maintenance.JobControl'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock'/>\n" +
             "  <component id='com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock'/>\n" +
-            "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesters'/>\n" +
+            "  <component id='com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud'/>\n" +
             "  <handler id='com.yahoo.vespa.hosted.controller.restapi.application.ApplicationApiHandler'>\n" +
             "    <binding>http://*/application/v4/*</binding>\n" +
             "  </handler>\n" +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index bc671d2375e..856b91b912e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -39,13 +39,15 @@ public class JobControllerApiHandlerHelperTest {
     private final ApplicationId appId = ApplicationId.from("vespa", "music", "default");
     private final Instant start = Instant.parse("2018-06-27T10:12:35Z");
 
+    private static Step lastStep = Step.values()[Step.values().length - 1];
+
     @Test
     public void jobTypeResponse() {
         Map<JobType, RunStatus> jobMap = new HashMap<>();
         List<JobType> jobList = new ArrayList<>();
-        jobMap.put(JobType.systemTest, createStatus(JobType.systemTest, 1, 30, Step.last(), Step.Status.succeeded));
+        jobMap.put(JobType.systemTest, createStatus(JobType.systemTest, 1, 30, lastStep, Step.Status.succeeded));
         jobList.add(JobType.systemTest);
-        jobMap.put(JobType.productionApNortheast1, createStatus(JobType.productionApNortheast1, 1, 60, Step.last(), Step.Status.succeeded));
+        jobMap.put(JobType.productionApNortheast1, createStatus(JobType.productionApNortheast1, 1, 60, lastStep, Step.Status.succeeded));
         jobList.add(JobType.productionApNortheast1);
         jobMap.put(JobType.productionUsWest1, createStatus(JobType.productionUsWest1, 1, 60, Step.startTests, Step.Status.failed));
         jobList.add(JobType.productionUsWest1);
@@ -61,13 +63,13 @@ public class JobControllerApiHandlerHelperTest {
         Map<RunId, RunStatus> statusMap = new HashMap<>();
         RunStatus status;
 
-        status = createStatus(JobType.systemTest, 3, 30, Step.last(), Step.Status.succeeded);
+        status = createStatus(JobType.systemTest, 3, 30, lastStep, Step.Status.succeeded);
         statusMap.put(status.id(), status);
 
         status = createStatus(JobType.systemTest, 2, 56, Step.installReal, Step.Status.failed);
         statusMap.put(status.id(), status);
 
-        status = createStatus(JobType.systemTest, 1, 44, Step.last(), Step.Status.succeeded);
+        status = createStatus(JobType.systemTest, 1, 44, lastStep, Step.Status.succeeded);
         statusMap.put(status.id(), status);
 
         URI jobTypeUrl = URI.create("https://domain.tld/application/v4/tenant/sometenant/application/someapp/instance/usuallydefault/job/systemtest");
@@ -85,7 +87,7 @@ public class JobControllerApiHandlerHelperTest {
         tester.curator().writeHistoricRuns(
                 runId.application(),
                 runId.type(),
-                Collections.singleton(createStatus(JobType.systemTest, 42, 44, Step.last(), Step.Status.succeeded)));
+                Collections.singleton(createStatus(JobType.systemTest, 42, 44, lastStep, Step.Status.succeeded)));
 
         logStore.append(runId, Step.deployTester.name(), "INFO\t1234567890\tSUCCESS".getBytes());
         logStore.append(runId, Step.installTester.name(), "INFO\t1234598760\tSUCCESS".getBytes());
@@ -114,7 +116,7 @@ public class JobControllerApiHandlerHelperTest {
         RunId runId = new RunId(appId, type, runid);
 
         Map<Step, Step.Status> stepStatusMap = new HashMap<>();
-        Arrays.stream(Step.values()).sorted(Comparator.naturalOrder()).forEach(step -> {
+        for (Step step : Step.values()) {
             if (step.ordinal() < lastStep.ordinal()) {
                 stepStatusMap.put(step, Step.Status.succeeded);
             } else if (step.equals(lastStep)) {
@@ -122,10 +124,10 @@ public class JobControllerApiHandlerHelperTest {
             } else {
                 stepStatusMap.put(step, Step.Status.unfinished);
             }
-        });
+        }
 
         Optional<Instant> end = Optional.empty();
-        if (lastStepStatus == Step.Status.failed || (lastStepStatus != Step.Status.unfinished && lastStep == Step.last())) {
+        if (lastStepStatus == Step.Status.failed || stepStatusMap.get(lastStep) == Step.Status.succeeded) {
             end = Optional.of(start.plusSeconds(duration));
         }