All green skies

11 files changed, 197 insertions, 78 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index cf5a0060c90..9594acc2931 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -1,5 +1,6 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
@@ -34,6 +35,7 @@ import static java.util.Comparator.naturalOrder;
 import static java.util.Objects.requireNonNull;
 import static java.util.stream.Collectors.collectingAndThen;
 import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toMap;
 import static java.util.stream.Collectors.toUnmodifiableList;
 import static java.util.stream.Collectors.toUnmodifiableMap;
 
@@ -82,12 +84,26 @@ public class DeploymentStatus {
     public Map<ApplicationId, JobList> instanceJobs() {
         return allJobs.asList().stream()
                       .collect(groupingBy(job -> job.id().application(),
-                                       collectingAndThen(toUnmodifiableList(), JobList::from)));
+                                          collectingAndThen(toUnmodifiableList(), JobList::from)));
     }
 
     /** Returns the set of jobs that need to run for the application's current change to be considered complete. */
     public Map<JobId, List<Versions>> jobsToRun() {
-        return jobsToRun(application().change());
+        Map<JobId, List<Versions>> jobs = jobsToRun(application().change());
+        if (application.outstandingChange().isEmpty())
+            return jobs;
+
+        // Add test jobs for any outstanding change.
+        var testJobs = jobsToRun(application.outstandingChange().onTopOf(application.change()))
+                .entrySet().stream()
+                .filter(entry -> ! entry.getKey().type().isProduction());
+
+        return Stream.concat(jobs.entrySet().stream(), testJobs)
+                     .collect(collectingAndThen(toMap(Map.Entry::getKey,
+                                                      Map.Entry::getValue,
+                                                      (l1, l2) -> ImmutableList.<Versions>builder().addAll(l1).addAll(l2).build(),
+                                                      LinkedHashMap::new),
+                                                ImmutableMap::copyOf));
     }
 
     /** Returns the set of jobs that need to run for the given change to be considered complete. */
@@ -96,8 +112,6 @@ public class DeploymentStatus {
 
         addProductionJobs(jobs, change);
         addTests(jobs);
-        if (jobs.isEmpty())
-            addTestsOnly(jobs, change);
 
         return ImmutableMap.copyOf(jobs);
     }
@@ -108,7 +122,7 @@ public class DeploymentStatus {
         steps.forEach((job, step) -> {
             Versions versions = Versions.from(change, application, deploymentFor(job), systemVersion);
             if (     job.type().isProduction()
-                &&   step.completedAt(change).isEmpty()
+                &&   step.completedAt(change, versions).isEmpty()
                 && ! step.isRunning(versions))
                 jobs.put(job, List.of(versions));
         });
@@ -167,13 +181,6 @@ public class DeploymentStatus {
                        .anyMatch(job -> job.type() == testJob && testJobs.get(job).containsAll(versions));
     }
 
-    private void addTestsOnly(Map<JobId, List<Versions>> jobs, Change change) {
-        /*steps.forEach((job, step) -> {
-            if (List.of(test, staging).contains(job.type().environment()))
-                jobs.put(job, List.of(Versions.from(change, application, Optional.empty(), systemVersion)));
-        });*/
-    }
-
     private  static <T> List<T> concat(List<T> first, List<T> second) {
         return Stream.concat(first.stream(), second.stream()).collect(toUnmodifiableList());
     }
@@ -199,7 +206,7 @@ public class DeploymentStatus {
     /** Adds the primitive steps contained in the given step, which depend on the given previous primitives, to the dependency graph. */
     List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, DeploymentSpec.Step step,
                               List<StepStatus> previous, InstanceName instance) {
-        if (step.steps().isEmpty()) { // TODO jonmv: Throw out empty container steps :(
+        if (step.steps().isEmpty()) {
             if ( ! step.delay().isZero())
                 return List.of(new DelayStatus((DeploymentSpec.Delay) step, previous));
 
@@ -209,6 +216,8 @@ public class DeploymentStatus {
                 jobType = JobType.from(system, ((DeclaredZone) step).environment(), null)
                                  .orElseThrow(() -> new IllegalStateException("No job is known for " + step + " in " + system));
                 stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, instance, jobType);
+                previous = new ArrayList<>(previous);
+                previous.add(stepStatus);
             }
             else if (step.isTest()) {
                 jobType = JobType.from(system, ((DeclaredTest) step).region())
@@ -295,12 +304,12 @@ public class DeploymentStatus {
         public final Optional<InstanceName> instance() { return instance; }
 
         /** The time at which this is complete on the given versions. */
-        public abstract Optional<Instant> completedAt(Change change);
+        public abstract Optional<Instant> completedAt(Change change, Versions versions);
 
         /** The time at which all dependencies completed on the given version. */
-        public Optional<Instant> readyAt(Change change) {
-            return dependencies.stream().allMatch(step -> step.completedAt(change).isPresent())
-                   ? dependencies.stream().map(step -> step.completedAt(change).get())
+        public Optional<Instant> readyAt(Change change, Versions versions) {
+            return dependencies.stream().allMatch(step -> step.completedAt(change, versions).isPresent())
+                   ? dependencies.stream().map(step -> step.completedAt(change, versions).get())
                                  .max(naturalOrder())
                                  .or(() -> Optional.of(Instant.EPOCH))
                    : Optional.empty();
@@ -318,8 +327,8 @@ public class DeploymentStatus {
         }
 
         @Override
-        public Optional<Instant> completedAt(Change change) {
-            return readyAt(change).map(completion -> completion.plus(step().delay()));
+        public Optional<Instant> completedAt(Change change, Versions versions) {
+            return readyAt(change, versions).map(completion -> completion.plus(step().delay()));
         }
 
         @Override
@@ -333,10 +342,12 @@ public class DeploymentStatus {
     public static abstract class JobStepStatus extends StepStatus {
 
         private final JobStatus job;
+        private final DeploymentStatus status;
 
-        protected JobStepStatus(DeploymentSpec.Step step, List<StepStatus> dependencies, JobStatus job) {
+        protected JobStepStatus(DeploymentSpec.Step step, List<StepStatus> dependencies, JobStatus job, DeploymentStatus status) {
             super(step, dependencies, job.id().application().instance());
             this.job = requireNonNull(job);
+            this.status = requireNonNull(status);
         }
 
         @Override
@@ -344,6 +355,34 @@ public class DeploymentStatus {
             return job.isRunning() && job.lastTriggered().get().versions().targetsMatch(versions);
         }
 
+        @Override
+        public Optional<Instant> readyAt(Change change, Versions versions) {
+            Optional<Instant> readyAt = super.readyAt(change, versions);
+            if (readyAt.isEmpty())
+                return Optional.empty();
+
+            Optional<Instant> pausedUntil = status.application.require(job.id().application().instance()).jobPause(job.id().type());
+            if (pausedUntil.isPresent() && pausedUntil.get().isAfter(readyAt.get()))
+                return pausedUntil;
+
+            if (job.lastTriggered().isEmpty()) return readyAt;
+            if (job.lastCompleted().isEmpty()) return readyAt;
+            if (job.firstFailing().isEmpty()) return readyAt;
+            if ( ! versions.targetsMatch(job.lastCompleted().get().versions())) return readyAt;
+            if (status.application.deploymentSpec().requireInstance(job.id().application().instance()).upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return readyAt;
+            if (job.id().type().environment().isTest() && job.isOutOfCapacity()) return readyAt;
+
+            Instant firstFailing = job.firstFailing().get().end().get();
+            Instant lastCompleted = job.lastCompleted().get().end().get();
+            if (lastCompleted.isBefore(readyAt.get()))
+                return readyAt;
+
+            return firstFailing.equals(lastCompleted) ? Optional.of(lastCompleted)
+                                                      : Optional.of(lastCompleted.plus(Duration.ofMinutes(10))
+                                                                                 .plus(Duration.between(firstFailing, lastCompleted)
+                                                                                               .dividedBy(2)));
+        }
+
         public static JobStepStatus ofProductionDeployment(DeclaredZone step, List<StepStatus> dependencies,
                                                            DeploymentStatus status, InstanceName instance, JobType jobType) {
             ZoneId zone = ZoneId.from(step.environment(), step.region().get());
@@ -351,37 +390,17 @@ public class DeploymentStatus {
             Optional<Deployment> existingDeployment = Optional.ofNullable(status.application().require(instance)
                                                                                 .deployments().get(zone));
 
-            return new JobStepStatus(step, dependencies, job) {
-                @Override
-                public Optional<Instant> readyAt(Change change) {
-                    Versions versions = Versions.from(change, status.application, existingDeployment, status.systemVersion);
-                    Optional<Instant> readyAt = super.readyAt(change)
-                                                     .filter(__ -> status.isTested(job.id(), versions));
-                    if (readyAt.isEmpty())
-                        return Optional.empty();
+            return new JobStepStatus(step, dependencies, job, status) {
 
-                    Optional<Instant> pausedUntil = status.application.require(instance).jobPause(jobType);
-                    if (pausedUntil.isPresent() && pausedUntil.get().isAfter(readyAt.get()))
-                        return pausedUntil;
-
-                    if (job.lastTriggered().isEmpty()) return readyAt;
-                    if (job.isSuccess()) return readyAt;
-                    if (job.lastCompleted().isEmpty()) return readyAt;
-                    if (job.firstFailing().isEmpty()) return readyAt;
-                    if ( ! versions.targetsMatch(job.lastCompleted().get().versions())) return readyAt;
-                    if (status.application.deploymentSpec().requireInstance(instance).upgradePolicy() == DeploymentSpec.UpgradePolicy.canary) return readyAt;
-                    if (jobType.environment().isTest() && job.isOutOfCapacity()) return readyAt;
-
-                    Instant firstFailing = job.firstFailing().get().end().get();
-                    Instant lastCompleted = job.lastCompleted().get().end().get();
-                    return Optional.of(lastCompleted.plus(Duration.ofMinutes(10))
-                                                    .plus(Duration.between(firstFailing, lastCompleted)
-                                                                  .dividedBy(2)));
+                @Override
+                public Optional<Instant> readyAt(Change change, Versions versions) {
+                    return super.readyAt(change, versions)
+                                .filter(__ -> status.isTested(job.id(), versions));
                 }
 
                 /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */
                 @Override
-                public Optional<Instant> completedAt(Change change) {
+                public Optional<Instant> completedAt(Change change, Versions versions) {
                     if (     change.isPinned()
                         &&   change.platform().isPresent()
                         && ! existingDeployment.map(Deployment::version).equals(change.platform()))
@@ -404,9 +423,9 @@ public class DeploymentStatus {
         public static JobStepStatus ofProductionTest(DeclaredTest step, List<StepStatus> dependencies,
                                                      DeploymentStatus status, InstanceName instance, JobType testType, JobType jobType) {
             JobStatus job = status.instanceJobs(instance).get(testType);
-            return new JobStepStatus(step, dependencies, job) {
+            return new JobStepStatus(step, dependencies, job, status) {
                 @Override
-                public Optional<Instant> completedAt(Change change) {
+                public Optional<Instant> completedAt(Change change, Versions versions) {
                     Versions toVerify = Versions.from(change, status.application.require(instance).deployments().get(ZoneId.from(prod, step.region())));
                     return job.lastSuccess()
                               .filter(run -> toVerify.targetsMatch(run.versions()))
@@ -419,11 +438,10 @@ public class DeploymentStatus {
 
         public static JobStepStatus ofTestDeployment(DeclaredZone step, List<StepStatus> dependencies,
                                                      DeploymentStatus status, InstanceName instance, JobType jobType) {
-            Versions versions = Versions.from(status.application, status.systemVersion);
             JobStatus job = status.instanceJobs(instance).get(jobType);
-            return new JobStepStatus(step, dependencies, job) {
+            return new JobStepStatus(step, dependencies, job, status) {
                 @Override
-                public Optional<Instant> completedAt(Change change) {
+                public Optional<Instant> completedAt(Change change, Versions versions) {
                     return RunList.from(job)
                                   .on(versions)
                                   .status(RunStatus.success)
@@ -479,6 +497,17 @@ public class DeploymentStatus {
      *
      * anySysTest && anyStaTest || triggeredProd
      *
+     *
+     *
+     *  Delay after test jobs? Requires test jobs in DAG,
+     *      which requires completeAt, and thus readyAt, to accept Change _and_ Versions. Public with Change only perhaps?
+     *      Careful not to require ALL tests to run with versions for each prod job!
+     *
+     *  readyAt and ordering of test jobs? Use lastSuccess.at for tests? Not quite right, but used today ...
+     *
+     *  Add all possible test jobs to steps. Add them as prerequisites for first job in each instance ... ?
+     *
      */
 
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index b5e8e3704db..b77f3a242c3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -263,11 +263,12 @@ public class DeploymentTrigger {
         List<Job> jobs = new ArrayList<>();
         applications().getApplication(id).map(controller.jobController()::deploymentStatus).ifPresent(status -> {
             status.jobsToRun().forEach((job, versionsList) -> {
-                status.stepStatus().get(job).readyAt(status.application().change()).ifPresent(readyAt -> {
                     for (Versions versions : versionsList)
+                        status.stepStatus().get(job).readyAt(status.application().change(), versions)
+                              .filter(readyAt -> ! clock.instant().isBefore(readyAt)).ifPresent(readyAt -> {
                         if ( ! (   isSuspendedInAnotherZone(status.application().require(job.application().instance()),
                                                             job.type().zone(controller.system()))
-                                && job.type().environment() != Environment.prod))
+                                && job.type().environment() == Environment.prod))
                             jobs.add(deploymentJob(status.application().require(job.application().instance()),
                                                    versions,
                                                    status.application().change(),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
index f7847d4974a..9b12bd77743 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java
@@ -72,6 +72,12 @@ import static java.util.stream.Collectors.toMap;
  */
 class JobControllerApiHandlerHelper {
 
+    static HttpResponse applicationJobs(Controller controller, TenantAndApplicationId id, URI baseUriForJobs) {
+        DeploymentStatus status = controller.jobController().deploymentStatus(controller.applications().requireApplication(id));
+
+        return null;
+    }
+
     /**
      * @return Response with all job types that have recorded runs for the application _and_ the status for the last run of that type
      */
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 70863c4ce3b..a6819cc2bae 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -103,7 +103,6 @@ public class ControllerTest {
 
         // system and staging test job - succeeding
         context.submit(applicationPackage);
-        applicationVersion = context.application().change().application().get();
         context.runJob(systemTest);
         context.runJob(stagingTest);
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 2e71cd62b5e..88e0a8b82b3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -610,31 +610,30 @@ public class DeploymentTriggerTest {
         // New application change is deployed and fails in system-test for a while
         app.submit(applicationPackage).runJob(stagingTest).failDeployment(systemTest);
 
-        // Retries immediately in the first minute after failing
-        tester.clock().advance(Duration.ofSeconds(59));
+        // Retries immediately once
         app.failDeployment(systemTest);
         tester.triggerJobs();
         app.assertRunning(systemTest);
 
-        // Stops immediate retry after failing for 1 minute
+        // Stops immediate retry when next triggering is considered after first failure
         tester.clock().advance(Duration.ofSeconds(1));
         app.failDeployment(systemTest);
         tester.triggerJobs();
         app.assertNotRunning(systemTest);
 
-        // Retries after 10 minutes since previous completion as we failed within the last hour
+        // Retries after 10 minutes since previous completion, plus half the time since the first failure
         tester.clock().advance(Duration.ofMinutes(10).plus(Duration.ofSeconds(1)));
         tester.triggerJobs();
         app.assertRunning(systemTest);
 
-        // Retries less frequently after 1 hour of failure
-        tester.clock().advance(Duration.ofMinutes(50));
+        // Retries less frequently as more time passes
         app.failDeployment(systemTest);
+        tester.clock().advance(Duration.ofMinutes(15));
         tester.triggerJobs();
         app.assertNotRunning(systemTest);
 
-        // Retries after two hours pass since last completion
-        tester.clock().advance(Duration.ofHours(2).plus(Duration.ofSeconds(1)));
+        // Retries again when sufficient time has passed
+        tester.clock().advance(Duration.ofSeconds(2));
         tester.triggerJobs();
         app.assertRunning(systemTest);
 
@@ -721,15 +720,12 @@ public class DeploymentTriggerTest {
         assertEquals(List.of(), tester.jobs().active());
 
         tester.readyJobsTrigger().maintain();
-        app2.assertRunning(stagingTest);
         assertEquals(1, tester.jobs().active().size());
 
         tester.readyJobsTrigger().maintain();
-        app1.assertRunning(stagingTest);
         assertEquals(2, tester.jobs().active().size());
 
         tester.readyJobsTrigger().maintain();
-        app3.assertRunning(stagingTest);
         assertEquals(3, tester.jobs().active().size());
 
         // Remove the jobs for app1 and app2, and then let app3 fail with outOfCapacity.
@@ -743,11 +739,9 @@ public class DeploymentTriggerTest {
         assertEquals(1, tester.jobs().active().size());
 
         tester.readyJobsTrigger().maintain();
-        app2.assertRunning(stagingTest);
         assertEquals(2, tester.jobs().active().size());
 
         tester.readyJobsTrigger().maintain();
-        app1.assertRunning(stagingTest);
         assertEquals(3, tester.jobs().active().size());
 
         // Finish deployment for apps 2 and 3, then release a new version, leaving only app1 with an application upgrade.
@@ -758,8 +752,6 @@ public class DeploymentTriggerTest {
 
         tester.controllerTester().upgradeSystem(new Version("6.2"));
         tester.upgrader().maintain();
-        // app1 also gets a new application change, so its time of availability is after the version upgrade.
-        tester.clock().advance(Duration.ofSeconds(1));
         app1.submit(applicationPackage);
         app1.jobAborted(stagingTest);
 
@@ -789,12 +781,12 @@ public class DeploymentTriggerTest {
         assertEquals(2, tester.jobs().active().size());
 
         tester.readyJobsTrigger().maintain();
-        app2.assertRunning(stagingTest);
         app1.assertRunning(systemTest);
         assertEquals(4, tester.jobs().active().size());
 
         tester.readyJobsTrigger().maintain();
         app3.assertRunning(stagingTest);
+        app2.assertRunning(stagingTest);
         app2.assertRunning(systemTest);
         assertEquals(6, tester.jobs().active().size());
     }
@@ -816,7 +808,7 @@ public class DeploymentTriggerTest {
         var app = tester.newDeploymentContext("tenant1", "application1", "instance1").submit(applicationPackage); // TODO jonmv: support instances in deployment context>
         var otherInstance = tester.newDeploymentContext("tenant1", "application1", "instance2");
         app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3);
-        otherInstance.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3);
+        otherInstance.runJob(productionUsEast3);
         assertEquals(2, app.application().instances().size());
         assertEquals(2, app.application().productionDeployments().values().stream()
                               .mapToInt(Collection::size)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index 80b54b154be..cfd4610c7b2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -875,7 +875,7 @@ public class UpgraderTest {
         tester.triggerJobs();
         assertEquals(3, tester.jobs().active().size()); // Just the running upgrade, and tests for the new revision.
 
-        app.runJob(systemTest).runJob(stagingTest).runJob(productionUsCentral1).runJob(productionUsEast3);
+        app.runJob(productionUsCentral1).runJob(productionUsEast3).runJob(systemTest).runJob(stagingTest);
         assertEquals(List.of(), tester.jobs().active()); // No jobs left.
 
         tester.outstandingChangeDeployer().run();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 766e64d20d0..7bbc7e91244 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -95,12 +95,20 @@ public class JobControllerApiHandlerHelperTest {
         assertEquals(running, tester.jobs().last(app.instanceId(), productionUsCentral1).get().status());
         assertEquals(running, tester.jobs().last(app.instanceId(), stagingTest).get().status());
 
-        // Staging deployment expires, the job fails, and won't be retried immediately.
+        // Staging deployment expires and the job fails, and is immediately retried.
+        tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone(tester.controller().system()));
+        tester.runner().run();
+        assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status());
+
+        // Staging deployment expires again, the job fails for the second time, and won't be retried immediately.
+        tester.clock().advance(Duration.ofMillis(100_000)); // Advance time to avoid immediate retry
+        tester.triggerJobs();
+        tester.runner().run();
+        assertEquals(running, tester.jobs().last(app.instanceId(), stagingTest).get().status());
         tester.controller().applications().deactivate(app.instanceId(), stagingTest.zone(tester.controller().system()));
         tester.runner().run();
         assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status());
 
-        tester.clock().advance(Duration.ofMillis(100_000)); // More than the minute within which there are immediate retries.
         tester.triggerJobs();
         assertEquals(installationFailed, tester.jobs().last(app.instanceId(), stagingTest).get().status());
 
@@ -113,8 +121,8 @@ public class JobControllerApiHandlerHelperTest {
         // us-central-1 has started, deployed, and is installing. Deployment is not yet verified.
         // us-east-3 is waiting for the failed staging test and us-central-1, while us-west-1 is waiting only for us-central-1.
         // Only us-east-3 is verified, on revision1.
-        // staging-test has 4 runs: one success without sources on revision1, one success from revision1 to revision2,
-        // one success from revision2 to revision3 and one failure from revision1 to revision3.
+        // staging-test has 5 runs: one success without sources on revision1, one success from revision1 to revision2,
+        // one success from revision2 to revision3 and two failures from revision1 to revision3.
         assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(app.instanceId(), stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json");
         assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(app.instanceId(), productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json");
         assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.controller(), app.instanceId(), URI.create("https://some.url:43/root/")), "overview.json");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
index d1e9a004dd2..c0f45543abb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json
@@ -239,6 +239,48 @@
           "reason": "Testing for productionUsEast3"
         },
         {
+          "id": 5,
+          "status": "installationFailed",
+          "start": 102000,
+          "end": 102000,
+          "wantedPlatform": "6.1",
+          "wantedApplication": {
+            "hash": "1.0.3-commit1",
+            "build": 3,
+            "source": {
+              "gitRepository": "repository1",
+              "gitBranch": "master",
+              "gitCommit": "commit1"
+            }
+          },
+          "currentPlatform": "6.1",
+          "currentApplication": {
+            "hash": "1.0.1-commit1",
+            "build": 1,
+            "source": {
+              "gitRepository": "repository1",
+              "gitBranch": "master",
+              "gitCommit": "commit1"
+            }
+          },
+          "steps": {
+            "deployTester": "succeeded",
+            "deployInitialReal": "succeeded",
+            "installInitialReal": "failed",
+            "deployReal": "unfinished",
+            "installTester": "unfinished",
+            "installReal": "unfinished",
+            "startTests": "unfinished",
+            "endTests": "unfinished",
+            "copyVespaLogs": "succeeded",
+            "deactivateReal": "succeeded",
+            "deactivateTester": "succeeded",
+            "report": "succeeded"
+          },
+          "tasks": {},
+          "log": "https://some.url:43/root/staging-test/run/5"
+        },
+        {
           "id": 4,
           "status": "installationFailed",
           "start": 2000,
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
index 68599618ab4..fbc480d6b03 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/staging-runs.json
@@ -168,5 +168,47 @@
     },
     "tasks": {},
     "log": "https://some.url:43/root/run/4"
+  },
+  "5": {
+    "id": 5,
+    "status": "installationFailed",
+    "start": 102000,
+    "end": 102000,
+    "wantedPlatform": "6.1",
+    "wantedApplication": {
+      "hash": "1.0.3-commit1",
+      "build": 3,
+      "source": {
+        "gitRepository": "repository1",
+        "gitBranch": "master",
+        "gitCommit": "commit1"
+      }
+    },
+    "currentPlatform": "6.1",
+    "currentApplication": {
+      "hash": "1.0.1-commit1",
+      "build": 1,
+      "source": {
+        "gitRepository": "repository1",
+        "gitBranch": "master",
+        "gitCommit": "commit1"
+      }
+    },
+    "steps": {
+      "deployInitialReal": "succeeded",
+      "installInitialReal": "failed",
+      "deployReal": "unfinished",
+      "installReal": "unfinished",
+      "deployTester": "succeeded",
+      "installTester": "unfinished",
+      "startTests": "unfinished",
+      "endTests": "unfinished",
+      "copyVespaLogs": "succeeded",
+      "deactivateReal": "succeeded",
+      "deactivateTester": "succeeded",
+      "report": "succeeded"
+    },
+    "tasks": {},
+    "log": "https://some.url:43/root/run/5"
   }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
index 8e55ec7dda8..b3cb74e9501 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java
@@ -50,7 +50,7 @@ public class DeploymentApiTest extends ControllerContainerTest {
         var appWithoutDeployments = deploymentTester.newDeploymentContext("tenant3", "application3", "default");
         failingApp.submit(applicationPackage).deploy();
         productionApp.submit(multiInstancePackage).runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1);
-        otherProductionApp.runJob(JobType.systemTest).runJob(JobType.stagingTest).runJob(JobType.productionUsWest1);
+        otherProductionApp.runJob(JobType.productionUsWest1);
 
         // Deploy once so that job information is stored, then remove the deployment
         appWithoutDeployments.submit(applicationPackage).deploy();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index ca5424696d2..d5c881bfe43 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -82,7 +82,7 @@
           "instance": "i2",
           "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2",
           "upgradePolicy": "default",
-          "running": "staging-test"
+          "running": "production-us-west-1"
         }
       ]
     }