Support multiple system/staging test cloud combinations

5 files changed, 161 insertions, 60 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index a8bd20cdfca..7904aa0c9d6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -33,6 +33,7 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -57,8 +58,11 @@ import static java.util.Comparator.naturalOrder;
 import static java.util.Objects.requireNonNull;
 import static java.util.function.BinaryOperator.maxBy;
 import static java.util.stream.Collectors.collectingAndThen;
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.mapping;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
+import static java.util.stream.Collectors.toSet;
 import static java.util.stream.Collectors.toUnmodifiableList;
 
 /**
@@ -74,8 +78,6 @@ public class DeploymentStatus {
 
     private final Application application;
     private final JobList allJobs;
-    private final JobType systemTest;
-    private final JobType stagingTest;
     private final VersionStatus versionStatus;
     private final Version systemVersion;
     private final Function<InstanceName, VersionCompatibility> versionCompatibility;
@@ -88,8 +90,6 @@ public class DeploymentStatus {
                             Version systemVersion, Function<InstanceName, VersionCompatibility> versionCompatibility, Instant now) {
         this.application = requireNonNull(application);
         this.zones = zones;
-        this.systemTest = JobType.systemTest(zones, null);
-        this.stagingTest = JobType.stagingTest(zones, null);
         this.versionStatus = requireNonNull(versionStatus);
         this.systemVersion = requireNonNull(systemVersion);
         this.versionCompatibility = versionCompatibility;
@@ -101,6 +101,14 @@ public class DeploymentStatus {
         this.allJobs = JobList.from(jobSteps.keySet().stream().map(allJobs).collect(toList()));
     }
 
+    private JobType systemTest(JobType dependent) {
+        return JobType.systemTest(zones, dependent == null ? null : findCloud(dependent));
+    }
+
+    private JobType stagingTest(JobType dependent) {
+        return JobType.stagingTest(zones, dependent == null ? null : findCloud(dependent));
+    }
+
     /** The application this deployment status concerns. */
     public Application application() {
         return application;
@@ -115,7 +123,7 @@ public class DeploymentStatus {
     private boolean hasFailures(StepStatus dependency, StepStatus dependent) {
         Set<StepStatus> dependents = new HashSet<>();
         fillDependents(dependency, new HashSet<>(), dependents, dependent);
-        Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(Collectors.toSet());
+        Set<JobId> criticalJobs = dependents.stream().flatMap(step -> step.job().stream()).collect(toSet());
 
         return ! allJobs.matching(job -> criticalJobs.contains(job.id()))
                         .failingHard()
@@ -208,16 +216,26 @@ public class DeploymentStatus {
             if (change == null || ! change.hasTargets())
                 return;
 
-            Optional<JobId> firstProductionJobWithDeployment = jobSteps.keySet().stream()
-                                                                       .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment())
-                                                                       .filter(jobId -> deploymentFor(jobId).isPresent())
-                                                                       .findFirst();
-            Versions versions = Versions.from(change,
-                                              application,
-                                              firstProductionJobWithDeployment.flatMap(this::deploymentFor),
-                                              fallbackPlatform(change, job));
-            if (step.completedAt(change, Optional.empty()).isEmpty())
-                jobs.merge(job, List.of(new Job(job.type(), versions, step.readyAt(change), change)), DeploymentStatus::union);
+            Collection<Optional<JobId>> firstProductionJobsWithDeployment = jobSteps.keySet().stream()
+                                                                                   .filter(jobId -> jobId.type().isProduction() && jobId.type().isDeployment())
+                                                                                   .filter(jobId -> deploymentFor(jobId).isPresent())
+                                                                                   .collect(groupingBy(jobId -> findCloud(jobId.type()),
+                                                                                                       Collectors.reducing((o, n) -> o))) // Take the first.
+                                                                                   .values();
+            if (firstProductionJobsWithDeployment.isEmpty())
+                firstProductionJobsWithDeployment = List.of(Optional.empty());
+
+            for (Optional<JobId> firstProductionJobWithDeploymentInCloud : firstProductionJobsWithDeployment) {
+                Versions versions = Versions.from(change,
+                                                  application,
+                                                  firstProductionJobWithDeploymentInCloud.flatMap(this::deploymentFor),
+                                                  fallbackPlatform(change, job));
+                if (step.completedAt(change, firstProductionJobWithDeploymentInCloud).isEmpty()) {
+                    JobType actualType = job.type().isSystemTest() ? systemTest(firstProductionJobWithDeploymentInCloud.map(JobId::type).orElse(null))
+                                                                   : stagingTest(firstProductionJobWithDeploymentInCloud.map(JobId::type).orElse(null));
+                    jobs.merge(job, List.of(new Job(actualType, versions, step.readyAt(change), change)), DeploymentStatus::union);
+                }
+            }
         });
         return Collections.unmodifiableMap(jobs);
     }
@@ -273,13 +291,33 @@ public class DeploymentStatus {
     }
 
     private <T extends Comparable<T>> Optional<T> newestTested(InstanceName instance, Function<Run, T> runMapper) {
-        return instanceJobs().get(application.id().instance(instance))
-                             .type(systemTest, stagingTest)
-                             .asList().stream().flatMap(jobs -> jobs.runs().values().stream())
-                             .filter(Run::hasSucceeded)
-                             .map(runMapper)
-                             .max(naturalOrder());
+        Set<CloudName> clouds = jobSteps.keySet().stream()
+                                        .filter(job -> job.type().isProduction())
+                                        .map(job -> findCloud(job.type()))
+                                        .collect(toSet());
+        List<ZoneId> testZones = new ArrayList<>();
+        if (application.deploymentSpec().requireInstance(instance).concerns(test)) {
+            if (clouds.isEmpty()) testZones.add(JobType.systemTest(zones, null).zone());
+            else for (CloudName cloud: clouds) testZones.add(JobType.systemTest(zones, cloud).zone());
+        }
+        if (application.deploymentSpec().requireInstance(instance).concerns(staging)) {
+            if (clouds.isEmpty()) testZones.add(JobType.stagingTest(zones, null).zone());
+            else for (CloudName cloud: clouds) testZones.add(JobType.stagingTest(zones, cloud).zone());
+        }
+
+        Map<ZoneId, Optional<T>> newestPerZone = instanceJobs().get(application.id().instance(instance))
+                                                               .type(systemTest(null), stagingTest(null))
+                                                               .asList().stream().flatMap(jobs -> jobs.runs().values().stream())
+                                                               .filter(Run::hasSucceeded)
+                                                               .collect(groupingBy(run -> run.id().type().zone(),
+                                                                                   mapping(runMapper, Collectors.maxBy(naturalOrder()))));
+        return newestPerZone.keySet().containsAll(testZones)
+               ? testZones.stream().map(newestPerZone::get)
+                          .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().compareTo(o.get()) < 0 ? n : o)
+                          .orElse(Optional.empty())
+               : Optional.empty();
     }
+
     /**
      * The change to a revision which all dependencies of the given instance has completed,
      * which does not downgrade any deployments in the instance,
@@ -344,8 +382,8 @@ public class DeploymentStatus {
                                                                         .filter(run -> run.versions().equals(versions))
                                                                         .findFirst())
                                                .map(Run::start);
-        Optional<Instant> systemTestedAt = testedAt(job.application(), systemTest, versions);
-        Optional<Instant> stagingTestedAt = testedAt(job.application(), stagingTest, versions);
+        Optional<Instant> systemTestedAt = testedAt(job.application(), systemTest(null), versions);
+        Optional<Instant> stagingTestedAt = testedAt(job.application(), stagingTest(null), versions);
         if (systemTestedAt.isEmpty() || stagingTestedAt.isEmpty()) return triggeredAt;
         Optional<Instant> testedAt = systemTestedAt.get().isAfter(stagingTestedAt.get()) ? systemTestedAt : stagingTestedAt;
         return triggeredAt.isPresent() && triggeredAt.get().isBefore(testedAt.get()) ? triggeredAt : testedAt;
@@ -354,14 +392,15 @@ public class DeploymentStatus {
     /** Earliest instant when versions were tested for the given instance */
     private Optional<Instant> testedAt(ApplicationId instance, JobType type, Versions versions) {
         return declaredTest(instance, type).map(__ -> allJobs.instance(instance.instance()))
-                                           .orElse(allJobs)
-                                           .type(type).asList().stream()
-                                           .flatMap(status -> RunList.from(status)
-                                                                        .on(versions)
-                                                                        .matching(Run::hasSucceeded)
-                                                                        .asList().stream()
-                                                                        .map(Run::start))
-                                           .min(naturalOrder());
+                                                    .orElse(allJobs)
+                                                    .type(type).asList().stream()
+                                                    .flatMap(status -> RunList.from(status)
+                                                                              .on(versions)
+                                                                              .matching(run -> run.id().type().zone().equals(type.zone()))
+                                                                              .matching(Run::hasSucceeded)
+                                                                              .asList().stream()
+                                                                              .map(Run::start))
+                                                    .min(naturalOrder());
     }
 
     private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) {
@@ -481,7 +520,7 @@ public class DeploymentStatus {
         // Both changes are ready for this step, and we look to the specified rollout to decide.
         boolean platformReadyFirst = platformReadyAt.get().isBefore(revisionReadyAt.get());
         boolean revisionReadyFirst = revisionReadyAt.get().isBefore(platformReadyAt.get());
-        boolean failingUpgradeOnlyTests = ! jobs().type(systemTest, stagingTest)
+        boolean failingUpgradeOnlyTests = ! jobs().type(systemTest(job.type()), stagingTest(job.type()))
                                                   .failingHardOn(Versions.from(change.withoutApplication(), application, deploymentFor(job), systemVersion))
                                                   .isEmpty();
         switch (rollout) {
@@ -504,12 +543,12 @@ public class DeploymentStatus {
     /** The test jobs that need to run prior to the given production deployment jobs. */
     public Map<JobId, List<Job>> testJobs(Map<JobId, List<Job>> jobs) {
         Map<JobId, List<Job>> testJobs = new LinkedHashMap<>();
-        for (JobType testType : List.of(systemTest, stagingTest)) {
-            jobs.forEach((job, versionsList) -> {
+        jobs.forEach((job, versionsList) -> {
+            for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) {
                 if (job.type().isProduction() && job.type().isDeployment()) {
                     declaredTest(job.application(), testType).ifPresent(testJob -> {
                         for (Job productionJob : versionsList)
-                            if (allJobs.successOn(productionJob.versions()).get(testJob).isEmpty())
+                            if (allJobs.successOn(testType, productionJob.versions()).asList().isEmpty())
                                 testJobs.merge(testJob, List.of(new Job(testJob.type(),
                                                                         productionJob.versions(),
                                                                         jobSteps().get(testJob).readyAt(productionJob.change),
@@ -517,13 +556,15 @@ public class DeploymentStatus {
                                                DeploymentStatus::union);
                     });
                 }
-            });
-            jobs.forEach((job, versionsList) -> {
+            }
+        });
+        jobs.forEach((job, versionsList) -> {
+            for (JobType testType : List.of(systemTest(job.type()), stagingTest(job.type()))) {
                 for (Job productionJob : versionsList)
                     if (   job.type().isProduction() && job.type().isDeployment()
-                        && allJobs.successOn(productionJob.versions()).type(testType).isEmpty()
+                        && allJobs.successOn(testType, productionJob.versions()).asList().isEmpty()
                         && testJobs.keySet().stream()
-                                   .noneMatch(test ->    test.type().equals(testType)
+                                   .noneMatch(test ->    test.type().equals(testType) && test.type().zone().equals(testType.zone())
                                                       && testJobs.get(test).stream().anyMatch(testJob -> testJob.versions().equals(productionJob.versions())))) {
                         JobId testJob = firstDeclaredOrElseImplicitTest(testType);
                         testJobs.merge(testJob,
@@ -533,8 +574,8 @@ public class DeploymentStatus {
                                                        productionJob.change)),
                                        DeploymentStatus::union);
                     }
-            });
-        }
+            }
+        });
         return Collections.unmodifiableMap(testJobs);
     }
 
@@ -597,7 +638,7 @@ public class DeploymentStatus {
             JobId jobId;
             StepStatus stepStatus;
             if (step.concerns(test) || step.concerns(staging)) {
-                jobType = step.concerns(test) ? systemTest : stagingTest;
+                jobType = step.concerns(test) ? systemTest(null) : stagingTest(null);
                 jobId = new JobId(application.id().instance(instance), jobType);
                 stepStatus = JobStepStatus.ofTestDeployment((DeclaredZone) step, List.of(), this, jobs.apply(jobId), true);
                 previous = new ArrayList<>(previous);
@@ -623,19 +664,19 @@ public class DeploymentStatus {
 
         if (step instanceof DeploymentInstanceSpec) {
             DeploymentInstanceSpec spec = ((DeploymentInstanceSpec) step);
-            StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name()));
+            StepStatus instanceStatus = new InstanceStatus(spec, previous, now, application.require(spec.name()), this);
             instance = spec.name();
             allSteps.add(instanceStatus);
             previous = List.of(instanceStatus);
             if (instance.equals(implicitSystemTest)) {
-                JobId job = new JobId(application.id().instance(instance), systemTest);
+                JobId job = new JobId(application.id().instance(instance), systemTest(null));
                 JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(test), List.of(),
                                                                           this, jobs.apply(job), false);
                 dependencies.put(job, testStatus);
                 allSteps.add(testStatus);
             }
             if (instance.equals(implicitStagingTest)) {
-                JobId job = new JobId(application.id().instance(instance), stagingTest);
+                JobId job = new JobId(application.id().instance(instance), stagingTest(null));
                 JobStepStatus testStatus = JobStepStatus.ofTestDeployment(new DeclaredZone(staging), List.of(),
                                                                           this, jobs.apply(job), false);
                 dependencies.put(job, testStatus);
@@ -776,13 +817,25 @@ public class DeploymentStatus {
         private final DeploymentInstanceSpec spec;
         private final Instant now;
         private final Instance instance;
+        private final DeploymentStatus status;
 
         private InstanceStatus(DeploymentInstanceSpec spec, List<StepStatus> dependencies, Instant now,
-                               Instance instance) {
+                               Instance instance, DeploymentStatus status) {
             super(StepType.instance, spec, dependencies, spec.name());
             this.spec = spec;
             this.now = now;
             this.instance = instance;
+            this.status = status;
+        }
+
+        /** The time at which this step is ready to run the specified change and / or versions. */
+        @Override
+        public Optional<Instant> readyAt(Change change) {
+            return status.jobSteps.keySet().stream()
+                                  .filter(job -> job.type().isProduction() && job.application().instance().equals(instance.name()))
+                                  .map(job -> super.readyAt(change, Optional.of(job)))
+                    .reduce((o, n) -> o.isEmpty() || n.isEmpty() ? Optional.empty() : n.get().isBefore(o.get()) ? n : o)
+                    .orElseGet(() -> super.readyAt(change, Optional.empty()));
         }
 
         /**
@@ -946,6 +999,7 @@ public class DeploymentStatus {
                                                             .orElseGet(() ->    (change.platform().isEmpty() || change.platform().get().equals(run.versions().targetPlatform()))
                                                                              && (change.revision().isEmpty() || change.revision().get().equals(run.versions().targetRevision()))))
                                   .matching(Run::hasSucceeded)
+                                  .matching(run -> dependent.isEmpty() || status.findCloud(dependent.get().type()).equals(status.findCloud(run.id().type())))
                                   .asList().stream()
                                   .map(run -> run.end().get())
                                   .max(naturalOrder());
@@ -960,16 +1014,22 @@ public class DeploymentStatus {
 
     public static class Job {
 
+        private final JobType type;
         private final Versions versions;
         private final Optional<Instant> readyAt;
         private final Change change;
 
         public Job(JobType type, Versions versions, Optional<Instant> readyAt, Change change) {
+            this.type = type;
             this.versions = type.isSystemTest() ? versions.withoutSources() : versions;
             this.readyAt = readyAt;
             this.change = change;
         }
 
+        public JobType type() {
+            return type;
+        }
+
         public Versions versions() {
             return versions;
         }
@@ -983,12 +1043,12 @@ public class DeploymentStatus {
             if (this == o) return true;
             if (o == null || getClass() != o.getClass()) return false;
             Job job = (Job) o;
-            return versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change);
+            return type.zone().equals(job.type.zone()) && versions.equals(job.versions) && readyAt.equals(job.readyAt) && change.equals(job.change);
         }
 
         @Override
         public int hashCode() {
-            return Objects.hash(versions, readyAt, change);
+            return Objects.hash(type.zone(), versions, readyAt, change);
         }
 
         @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index df8ee910dc4..fd1197312e8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -381,7 +381,7 @@ public class DeploymentTrigger {
                         .filter(__ -> abortIfRunning(status, jobsToRun, job)) // Abort and trigger this later if running with outdated parameters.
                         .map(readyAt -> deploymentJob(status.application().require(job.application().instance()),
                                                       versionsList.get(0).versions(),
-                                                      job.type(),
+                                                      versionsList.get(0).type(),
                                                       status.instanceJobs(job.application().instance()).get(job.type()),
                                                       readyAt))
                         .ifPresent(jobs::add);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
index 387ea755414..551f841233e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
@@ -119,8 +119,12 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> {
     }
 
     /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */
-    public JobList successOn(Versions versions) {
-        return matching(job -> ! RunList.from(job).matching(Run::hasSucceeded).on(versions).isEmpty());
+    public JobList successOn(JobType type, Versions versions) {
+        return matching(job ->      job.id().type().equals(type)
+                               && ! RunList.from(job)
+                                           .matching(run -> run.hasSucceeded() && run.id().type().zone().equals(type.zone()))
+                                           .on(versions)
+                                           .isEmpty());
     }
 
     // ----------------------------------- JobRun filtering
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 347f1d4ab15..ad236fcd4de 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -384,13 +384,13 @@ public class DeploymentContext {
     /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */
     public DeploymentContext completeRollout(boolean multiInstance) {
         triggerJobs();
-        Map<ApplicationId, Map<JobType, Versions>> jobsByInstance = new HashMap<>();
+        Map<ApplicationId, Map<JobType, Run>> runsByInstance = new HashMap<>();
         List<Run> activeRuns;
         while ( ! (activeRuns = this.jobs.active(applicationId)).isEmpty())
             for (Run run : activeRuns) {
-                Map<JobType, Versions> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashMap<>());
-                Versions previous = jobs.put(run.id().type(), run.versions());
-                if (run.versions().equals(previous)) {
+                Map<JobType, Run> runs = runsByInstance.computeIfAbsent(run.id().application(), k -> new HashMap<>());
+                Run previous = runs.put(run.id().type(), run);
+                if (previous != null && run.versions().equals(previous.versions()) && run.id().type().zone().equals(previous.id().type().zone())) {
                     throw new AssertionError("Job '" + run.id() + "' was run twice on same versions");
                 }
                 runJob(run.id().type(), run.id().application());
@@ -451,8 +451,8 @@ public class DeploymentContext {
 
     /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */
     private DeploymentContext runJob(JobType type, ApplicationId instance) {
-        var job = new JobId(instance, type);
         triggerJobs();
+        var job = currentRun(new JobId(instance, type)).id().job();
         doDeploy(job);
         if (job.type().isDeployment()) {
             doUpgrade(job);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index dd9c07bafde..e5000a85f71 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -2212,13 +2212,29 @@ public class DeploymentTriggerTest {
         DeploymentContext main = tester.newDeploymentContext("tenant", "application", "main");
         Version version1 = new Version("7");
         tester.controllerTester().upgradeSystem(version1);
-        tests.submit(appPackage).deploy();
+        tests.submit(appPackage);
         Optional<RevisionId> revision1 = tests.lastSubmission();
         JobId systemTestJob = new JobId(tests.instanceId(), systemTest);
         JobId stagingTestJob = new JobId(tests.instanceId(), stagingTest);
         JobId mainJob = new JobId(main.instanceId(), productionUsEast3);
         JobId centauriJob = new JobId(main.instanceId(), JobType.deploymentTo(prodAlphaCentauri.getId()));
 
+        assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet());
+        tests.runJob(systemTest).runJob(stagingTest).triggerJobs();
+
+        assertEquals(Set.of(systemTestJob, stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet());
+        tests.triggerJobs();
+        assertEquals(3, tester.jobs().active().size());
+
+        tests.runJob(systemTest);
+        tester.outstandingChangeDeployer().run();
+
+        assertEquals(2, tester.jobs().active().size());
+        main.assertRunning(productionUsEast3);
+
+        tests.runJob(stagingTest);
+        main.runJob(productionUsEast3).runJob(centauriJob.type());
+
         assertEquals(Change.empty(), tests.instance().change());
         assertEquals(Change.empty(), main.instance().change());
         assertEquals(Set.of(), tests.deploymentStatus().jobsToRun().keySet());
@@ -2235,19 +2251,21 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(version2), tests.instance().change());
         assertEquals(Change.empty(), main.instance().change());
         assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet());
+        assertEquals(2, tests.deploymentStatus().jobsToRun().get(systemTestJob).size());
 
         Version version3 = new Version("9");
         tester.controllerTester().upgradeSystem(version3);
-        tests.failDeployment(systemTest);
+        tests.runJob(systemTest)            // Success in default cloud.
+             .failDeployment(systemTest);   // Failure in centauri cloud.
         tester.upgrader().run();
 
         assertEquals(Change.of(version3), tests.instance().change());
         assertEquals(Change.empty(), main.instance().change());
         assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet());
 
-        tests.runJob(systemTest);
+        tests.runJob(systemTest).runJob(systemTest);
         tester.upgrader().run();
-        tests.runJob(stagingTest);
+        tests.runJob(stagingTest).runJob(stagingTest);
 
         assertEquals(Change.empty(), tests.instance().change());
         assertEquals(Change.of(version3), main.instance().change());
@@ -2276,6 +2294,7 @@ public class DeploymentTriggerTest {
         assertEquals(Change.of(revision2.get()), tests.instance().change());
         assertEquals(Change.empty(), main.instance().change());
         assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet());
+        assertEquals(2, tests.deploymentStatus().jobsToRun().get(systemTestJob).size());
 
         tests.submit(appPackage);
         Optional<RevisionId> revision3 = tests.lastSubmission();
@@ -2292,8 +2311,26 @@ public class DeploymentTriggerTest {
         assertEquals(Set.of(systemTestJob), tests.deploymentStatus().jobsToRun().keySet());
 
         tests.runJob(systemTest);
+        assertEquals(Change.of(revision3.get()), tests.instance().change());
+        assertEquals(Change.empty(), main.instance().change());
+        assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet());
+
+        tester.outstandingChangeDeployer().run();
+        tester.outstandingChangeDeployer().run();
+        tests.runJob(stagingTest);
+
+        assertEquals(Change.of(revision3.get()), tests.instance().change());
+        assertEquals(Change.empty(), main.instance().change());
+        assertEquals(Set.of(systemTestJob, stagingTestJob), tests.deploymentStatus().jobsToRun().keySet());
+
+        tests.runJob(systemTest);
         tester.outstandingChangeDeployer().run();
         tester.outstandingChangeDeployer().run();
+
+        assertEquals(Change.empty(), tests.instance().change());
+        assertEquals(Change.of(revision3.get()), main.instance().change());
+        assertEquals(Set.of(stagingTestJob, mainJob, centauriJob), tests.deploymentStatus().jobsToRun().keySet());
+
         tests.runJob(stagingTest);
 
         assertEquals(Change.empty(), tests.instance().change());