Merge pull request #3283 from vespa-engine/mpolden/trigger-deployments-in-parallel

Parallel deployments

11 files changed, 409 insertions, 216 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index 61231404a94..02e0d94920e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.application;
 
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
@@ -16,11 +15,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.stream.Collectors;
 
 /**
  * Information about which deployment jobs an application should run and their current status.
@@ -123,12 +120,27 @@ public class DeploymentJobs {
             return true;
         }
         if (environment == Environment.staging) {
-            return isSuccessful(JobType.systemTest, change.get());
+            return isSuccessful(change.get(), JobType.systemTest);
         } else if (environment == Environment.prod) {
-            return isSuccessful(JobType.stagingTest, change.get());
+            return isSuccessful(change.get(), JobType.stagingTest);
         }
         return true; // other environments do not have any preconditions
     }
+
+    /** Returns whether change has been deployed completely */
+    public boolean isDeployed(Change change) {
+        return status.values().stream()
+                .filter(status -> status.type().isProduction())
+                .allMatch(status -> isSuccessful(change, status.type()));
+    }
+
+    /** Returns whether job has completed successfully */
+    public boolean isSuccessful(Change change, JobType jobType) {
+        return Optional.ofNullable(jobStatus().get(jobType))
+                .filter(JobStatus::isSuccess)
+                .filter(status -> status.lastCompletedFor(change))
+                .isPresent();
+    }
     
     /** Returns the oldest failingSince time of the jobs of this, or null if none are failing */
     public Instant failingSince() {
@@ -150,13 +162,6 @@ public class DeploymentJobs {
 
     public Optional<String> jiraIssueId() { return jiraIssueId; }
 
-    private boolean isSuccessful(JobType jobType, Change change) {
-        return Optional.ofNullable(jobStatus().get(jobType))
-                .filter(JobStatus::isSuccess)
-                .filter(status -> status.lastCompletedFor(change))
-                .isPresent();
-    }
-
     /** Job types that exist in the build system */
     public enum JobType {
 
@@ -252,14 +257,6 @@ public class DeploymentJobs {
             return from(system, new com.yahoo.config.provision.Zone(environment, region));
         }
 
-        /** Returns the trigger order to use according to deployment spec */
-        public static List<JobType> triggerOrder(SystemName system, DeploymentSpec deploymentSpec) {
-            return deploymentSpec.zones().stream()
-                    .map(declaredZone -> JobType.from(system, declaredZone.environment(),
-                                                      declaredZone.region().orElse(null)))
-                    .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
-        }
-
         private static Zone zone(SystemName system, String environment, String region) {
             return new Zone(system, Environment.from(environment), RegionName.from(region));
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
new file mode 100644
index 00000000000..448ab419853
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
@@ -0,0 +1,162 @@
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
+import com.yahoo.vespa.hosted.controller.application.JobStatus;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.collectingAndThen;
+
+/**
+ * This class determines the order of deployments according to an application's deployment spec.
+ *
+ * @author mpolden
+ */
+public class DeploymentOrder {
+
+    private static final Logger log = Logger.getLogger(DeploymentOrder.class.getName());
+
+    private final Controller controller;
+    private final Clock clock;
+
+    public DeploymentOrder(Controller controller) {
+        Objects.requireNonNull(controller, "controller cannot be null");
+        this.controller = controller;
+        this.clock = controller.clock();
+    }
+
+    /** Returns a list of jobs to trigger after the given job */
+    public List<JobType> nextAfter(JobType job, Application application) {
+        // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a
+        // new application with no previous deployments)
+        if (job == JobType.component) {
+            return Collections.singletonList(JobType.systemTest);
+        }
+
+        // At this point we have deployed to system test, so deployment spec is available
+        List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application);
+        Optional<DeploymentSpec.Step> currentStep = fromJob(job, application);
+        if ( ! currentStep.isPresent()) {
+            return Collections.emptyList();
+        }
+
+        // If this is the last deployment step there's nothing more to trigger
+        int currentIndex = deploymentSteps.indexOf(currentStep.get());
+        if (currentIndex == deploymentSteps.size() - 1) {
+            return Collections.emptyList();
+        }
+
+        // Postpone if step hasn't completed all it's jobs for this change
+        if (!completedSuccessfully(currentStep.get(), application)) {
+            return Collections.emptyList();
+        }
+
+        // Postpone next job if delay has not passed yet
+        Duration delay = delayAfter(currentStep.get(), application);
+        if (postponeDeployment(delay, job, application)) {
+            log.info(String.format("Delaying next job after %s of %s by %s", job, application, delay));
+            return Collections.emptyList();
+        }
+
+        DeploymentSpec.Step nextStep = deploymentSteps.get(currentIndex + 1);
+        return nextStep.zones().stream()
+                .map(this::toJob)
+                .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
+    /** Returns whether the given job is first in a deployment */
+    public boolean isFirst(JobType job) {
+        return job == JobType.component;
+    }
+
+    /** Returns whether the given job is last in a deployment */
+    public boolean isLast(JobType job, Application application) {
+        List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application);
+        if (deploymentSteps.isEmpty()) { // Deployment spec not yet available
+            return false;
+        }
+        DeploymentSpec.Step lastStep = deploymentSteps.get(deploymentSteps.size() - 1);
+        return fromJob(job, application).get().equals(lastStep);
+    }
+
+    /** Returns jobs for given deployment spec, in the order they are declared */
+    public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) {
+        return deploymentSpec.steps().stream()
+                .flatMap(step -> jobsFrom(step).stream())
+                .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
+    /** Returns jobs for the given step */
+    private List<JobType> jobsFrom(DeploymentSpec.Step step) {
+        return step.zones().stream()
+                .map(this::toJob)
+                .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+    }
+
+    /** Returns whether all jobs have completed successfully for given step */
+    private boolean completedSuccessfully(DeploymentSpec.Step step, Application application) {
+        return jobsFrom(step).stream()
+                .allMatch(job -> application.deploymentJobs().isSuccessful(application.deploying().get(), job));
+    }
+
+    /** Resolve deployment step from job */
+    private Optional<DeploymentSpec.Step> fromJob(JobType job, Application application) {
+        for (DeploymentSpec.Step step : application.deploymentSpec().steps()) {
+            if (step.deploysTo(job.environment(), job.isProduction() ? job.region(controller.system()) : Optional.empty())) {
+                return Optional.of(step);
+            }
+        }
+        return Optional.empty();
+    }
+
+    /** Resolve job from deployment step */
+    private JobType toJob(DeploymentSpec.DeclaredZone zone) {
+        return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null));
+    }
+
+    /** Returns whether deployment should be postponed according to delay */
+    private boolean postponeDeployment(Duration delay, JobType job, Application application) {
+        Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(job))
+                .flatMap(JobStatus::lastSuccess)
+                .map(JobStatus.JobRun::at);
+        return lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant());
+    }
+
+    /** Find all steps that deploy to one or more zones */
+    private static List<DeploymentSpec.Step> deploymentSteps(Application application) {
+        return application.deploymentSpec().steps().stream()
+                .filter(step -> step instanceof DeploymentSpec.DeclaredZone ||
+                                step instanceof DeploymentSpec.ParallelZones)
+                .collect(Collectors.toList());
+    }
+
+    /** Determines the delay that should pass after the given step */
+    private static Duration delayAfter(DeploymentSpec.Step step, Application application) {
+        int stepIndex = application.deploymentSpec().steps().indexOf(step);
+        if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) {
+            return Duration.ZERO;
+        }
+        Duration totalDelay = Duration.ZERO;
+        List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps()
+                .subList(stepIndex + 1, application.deploymentSpec().steps().size());
+        for (DeploymentSpec.Step s : remainingSteps) {
+            if (!(s instanceof DeploymentSpec.Delay)) {
+                break;
+            }
+            totalDelay = totalDelay.plus(((DeploymentSpec.Delay) s).duration());
+        }
+        return totalDelay;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index ce1fee4dd14..5f529d21995 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -39,6 +39,7 @@ public class DeploymentTrigger {
     private final Controller controller;
     private final Clock clock;
     private final BuildSystem buildSystem;
+    private final DeploymentOrder order;
 
     public DeploymentTrigger(Controller controller, CuratorDb curator, Clock clock) {
         Objects.requireNonNull(controller,"controller cannot be null");
@@ -46,6 +47,7 @@ public class DeploymentTrigger {
         this.controller = controller;
         this.clock = clock;
         this.buildSystem = new PolledBuildSystem(controller, curator);
+        this.order = new DeploymentOrder(controller);
     }
     
     //--- Start of methods which triggers deployment jobs -------------------------
@@ -62,7 +64,7 @@ public class DeploymentTrigger {
             application = application.withJobCompletion(report, clock.instant(), controller);
             
             // Handle successful first and last job
-            if (isFirstJob(report.jobType()) && report.success()) { // the first job tells us that a change occurred
+            if (order.isFirst(report.jobType()) && report.success()) { // the first job tells us that a change occurred
                 if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) { // postpone until the current deployment is done
                     applications().store(application.withOutstandingChange(true), lock);
                     return;
@@ -70,17 +72,17 @@ public class DeploymentTrigger {
                 else { // start a new change deployment
                     application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown()));
                 }
-            } else if (isLastJob(report.jobType(), application) && report.success()) {
+            } else if (order.isLast(report.jobType(), application) && report.success() && application.deploymentJobs().isDeployed(application.deploying().get())) {
                 application = application.withDeploying(Optional.empty());
             }
 
             // Trigger next
             if (report.success())
-                application = trigger(nextAfter(report.jobType(), application), application, report.jobType() + " completed successfully", lock);
+                application = trigger(order.nextAfter(report.jobType(), application), application, report.jobType() + " completed successfully", lock);
             else if (isCapacityConstrained(report.jobType()) && shouldRetryOnOutOfCapacity(application, report.jobType()))
                 application = trigger(report.jobType(), application, true, "Retrying due to out of capacity", lock);
             else if (shouldRetryNow(application))
-                application = trigger(report.jobType(), application, "Retrying as job just started failing", lock);
+                application = trigger(report.jobType(), application, false, "Retrying as job just started failing", lock);
 
             applications().store(application, lock);
         }
@@ -95,15 +97,15 @@ public class DeploymentTrigger {
             if (shouldRetryFromBeginning(application)) {
                 // failed for a long time: Discard existing change and restart from the component job
                 application = application.withDeploying(Optional.empty());
-                application = trigger(JobType.component, application, "Retrying failing deployment from beginning: " + cause, lock);
+                application = trigger(JobType.component, application, false, "Retrying failing deployment from beginning: " + cause, lock);
                 applications().store(application, lock);
             } else {
                 // retry the failed job (with backoff)
-                for (JobType jobType : JobType.triggerOrder(controller.system(), application.deploymentSpec())) { // retry the *first* failing job
+                for (JobType jobType : order.jobsFrom(application.deploymentSpec())) { // retry the *first* failing job
                     JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
                     if (isFailing(jobStatus)) {
                         if (shouldRetryNow(jobStatus)) {
-                            application = trigger(jobType, application, "Retrying failing job: " + cause, lock);
+                            application = trigger(jobType, application, false, "Retrying failing job: " + cause, lock);
                             applications().store(application, lock);
                         }
                         break;
@@ -133,7 +135,7 @@ public class DeploymentTrigger {
             // Trigger next
             try (Lock lock = applications().lock(application.id())) {
                 application = applications().require(application.id());
-                application = trigger(nextAfter(lastSuccessfulJob.get().type(), application), application,
+                application = trigger(order.nextAfter(lastSuccessfulJob.get().type(), application), application,
                                       "Resuming delayed deployment", lock);
                 applications().store(application, lock);
             }
@@ -154,7 +156,7 @@ public class DeploymentTrigger {
             application = application.withDeploying(Optional.of(change));
             if (change instanceof Change.ApplicationChange)
                 application = application.withOutstandingChange(false);
-            application = trigger(JobType.systemTest, application, "Deploying change", lock);
+            application = trigger(JobType.systemTest, application, false, "Deploying change", lock);
             applications().store(application, lock);
         }
     }
@@ -177,77 +179,6 @@ public class DeploymentTrigger {
 
     private ApplicationController applications() { return controller.applications(); }
     
-    /** Returns the next job to trigger after this job, or null if none should be triggered */
-    private JobType nextAfter(JobType jobType, Application application) {
-        // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a
-        // new application with no previous deployments)
-        if (jobType == JobType.component) {
-            return JobType.systemTest;
-        }
-
-        // At this point we've at least deployed to system test, so deployment spec should be available
-        List<DeploymentSpec.DeclaredZone> zones = application.deploymentSpec().zones();
-        Optional<DeploymentSpec.DeclaredZone> zoneForJob = zoneForJob(application, jobType);
-        if (!zoneForJob.isPresent()) {
-            return null;
-        }
-        int zoneIndex = application.deploymentSpec().zones().indexOf(zoneForJob.get());
-
-        // This is last zone
-        if (zoneIndex == zones.size() - 1) {
-            return null;
-        }
-
-        // Skip next job if delay has not passed yet
-        Duration delay = delayAfter(application, zoneForJob.get());
-        Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType))
-                .flatMap(JobStatus::lastSuccess)
-                .map(JobStatus.JobRun::at);
-        if (lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant())) {
-            log.info(String.format("Delaying next job after %s of %s by %s", jobType, application, delay));
-            return null;
-        }
-
-        DeploymentSpec.DeclaredZone nextZone = application.deploymentSpec().zones().get(zoneIndex + 1);
-        return JobType.from(controller.system(), nextZone.environment(), nextZone.region().orElse(null));
-    }
-
-    private Duration delayAfter(Application application, DeploymentSpec.DeclaredZone zone) {
-        int stepIndex = application.deploymentSpec().steps().indexOf(zone);
-        if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) {
-            return Duration.ZERO;
-        }
-        Duration totalDelay = Duration.ZERO;
-        List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps()
-                .subList(stepIndex + 1, application.deploymentSpec().steps().size());
-        for (DeploymentSpec.Step step : remainingSteps) {
-            if (!(step instanceof DeploymentSpec.Delay)) {
-                break;
-            }
-            totalDelay = totalDelay.plus(((DeploymentSpec.Delay) step).duration());
-        }
-        return totalDelay;
-    }
-
-    private Optional<DeploymentSpec.DeclaredZone> zoneForJob(Application application, JobType jobType) {
-        return application.deploymentSpec()
-                .zones()
-                .stream()
-                .filter(zone -> zone.deploysTo(
-                        jobType.environment(),
-                        jobType.isProduction() ? jobType.region(controller.system()) : Optional.empty()))
-                .findFirst();
-    }
-
-    private boolean isFirstJob(JobType jobType) {
-        return jobType == JobType.component;
-    }
-
-    private boolean isLastJob(JobType jobType, Application application) {
-        List<JobType> triggerOrder = JobType.triggerOrder(controller.system(), application.deploymentSpec());
-        return triggerOrder.isEmpty() || jobType.equals(triggerOrder.get(triggerOrder.size() - 1));
-    }
-    
     private boolean isFailing(JobStatus jobStatusOrNull) {
         return jobStatusOrNull != null && !jobStatusOrNull.isSuccess();
     }
@@ -356,8 +287,11 @@ public class DeploymentTrigger {
         return application.withJobTriggering(jobType, clock.instant(), controller);
     }
 
-    private Application trigger(JobType jobType, Application application, String cause, Lock lock) {
-        return trigger(jobType, application, false, cause, lock);
+    private Application trigger(List<JobType> jobs, Application application, String cause, Lock lock) {
+        for (JobType job : jobs) {
+            application = trigger(job, application, false, cause, lock);
+        }
+        return application;
     }
 
     public BuildSystem buildSystem() { return buildSystem; }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 3dddfaf58a1..29b34747573 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -104,8 +104,8 @@ public class ControllerTest {
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
         assertFalse("Revision is currently not known",
                     ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent());
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
         assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
 
         Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision();
@@ -120,7 +120,7 @@ public class ControllerTest {
         tester.clock().advance(Duration.ofSeconds(1));
 
         // production job (failing)
-        tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, false);
+        tester.deployAndNotify(app1, applicationPackage, false, productionCorpUsEast1);
         assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
 
         JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1)
@@ -144,14 +144,14 @@ public class ControllerTest {
 
         // system and staging test job - succeeding
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
         assertStatus(JobStatus.initial(systemTest)
                               .withTriggering(version1, revision, tester.clock().instant())
                               .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller());
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
 
         // production job succeeding now
-        tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1);
         expectedJobStatus = expectedJobStatus
                 .withTriggering(version1, revision, tester.clock().instant())
                 .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller());
@@ -161,7 +161,7 @@ public class ControllerTest {
         assertStatus(JobStatus.initial(productionUsEast3)
                               .withTriggering( version1, revision, tester.clock().instant()), 
                      app1.id(), tester.controller());
-        tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
 
         assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size());
         
@@ -189,7 +189,7 @@ public class ControllerTest {
                 .environment(Environment.prod)
                 .region("us-east-3")
                 .build();
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
         assertNull("Zone was removed",
                    applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get()));
         assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1));
@@ -211,9 +211,9 @@ public class ControllerTest {
 
         // First deployment: An application change
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
 
         app1 = applications.require(app1.id());
         assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get());
@@ -234,9 +234,9 @@ public class ControllerTest {
                 .region("us-east-3")
                 .build();
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
 
         app1 = applications.require(app1.id());
         assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
@@ -248,7 +248,7 @@ public class ControllerTest {
                 .region("us-west-1")
                 .region("us-east-3")
                 .build();
-        tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
         app1 = applications.require(app1.id());
         assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
         assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get());
@@ -256,10 +256,10 @@ public class ControllerTest {
         // Version upgrade changes system version
         Change.VersionChange change = new Change.VersionChange(newSystemVersion);
         applications.deploymentTrigger().triggerChange(app1.id(), change);
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
 
         app1 = applications.require(app1.id());
         assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get());
@@ -326,37 +326,37 @@ public class ControllerTest {
         // Initial failure
         Instant initialFailure = tester.clock().instant();
         tester.notifyJobCompletion(component, app, true);
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at initial failure", 
                      initialFailure, firstFailing(app, tester).get().at());
 
         // Failure again -- failingSince should remain the same
         tester.clock().advance(Duration.ofMillis(1000));
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at second consecutive failure", 
                      initialFailure, firstFailing(app, tester).get().at());
 
         // Success resets failingSince
         tester.clock().advance(Duration.ofMillis(1000));
-        tester.deployAndNotify(systemTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, systemTest);
         assertFalse(firstFailing(app, tester).isPresent());
 
         // Complete deployment
-        tester.deployAndNotify(stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, productionCorpUsEast1);
         
         // Two repeated failures again.
         // Initial failure
         tester.clock().advance(Duration.ofMillis(1000));
         initialFailure = tester.clock().instant();
         tester.notifyJobCompletion(component, app, true);
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at initial failure", 
                      initialFailure, firstFailing(app, tester).get().at());
 
         // Failure again -- failingSince should remain the same
         tester.clock().advance(Duration.ofMillis(1000));
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at second consecutive failure", 
                      initialFailure, firstFailing(app, tester).get().at());
     }
@@ -435,11 +435,11 @@ public class ControllerTest {
 
         // foo: passes system test
         tester.notifyJobCompletion(component, foo, true);
-        tester.deployAndNotify(systemTest, foo, applicationPackage, true);
+        tester.deployAndNotify(foo, applicationPackage, true, systemTest);
 
         // bar: passes system test
         tester.notifyJobCompletion(component, bar, true);
-        tester.deployAndNotify(systemTest, bar, applicationPackage, true);
+        tester.deployAndNotify(bar, applicationPackage, true, systemTest);
 
         // foo and bar: staging test jobs queued
         assertEquals(2, buildSystem.jobs().size());
@@ -455,14 +455,14 @@ public class ControllerTest {
         }
 
         // bar: Completes deployment
-        tester.deployAndNotify(stagingTest, bar, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, bar, applicationPackage, true);
+        tester.deployAndNotify(bar, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(bar, applicationPackage, true, productionCorpUsEast1);
 
         // foo: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by
         // out of capacity retry mechanism
         tester.clock().advance(Duration.ofMinutes(15));
         tester.notifyJobCompletion(component, foo, true);
-        tester.deployAndNotify(systemTest, foo, applicationPackage, true);
+        tester.deployAndNotify(foo, applicationPackage, true, systemTest);
         tester.deploy(stagingTest, foo, applicationPackage);
         assertEquals(1, buildSystem.takeJobsToRun().size());
         tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity));
@@ -470,7 +470,7 @@ public class ControllerTest {
 
         // bar: New change triggers another staging-test job
         tester.notifyJobCompletion(component, bar, true);
-        tester.deployAndNotify(systemTest, bar, applicationPackage, true);
+        tester.deployAndNotify(bar, applicationPackage, true, systemTest);
         assertEquals(1, buildSystem.jobs().size());
 
         // foo: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index aa115421f6a..23451c60f08 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -13,6 +13,7 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -45,6 +46,13 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
+    public ApplicationPackageBuilder parallel(String... regionName) {
+        environmentBody.append("    <parallel>\n");
+        Arrays.stream(regionName).forEach(this::region);
+        environmentBody.append("    </parallel>\n");
+        return this;
+    }
+
     public ApplicationPackageBuilder delay(Duration delay) {
         environmentBody.append("    <delay seconds='");
         environmentBody.append(delay.getSeconds());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 0e816be864d..9b05101b5eb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
@@ -23,7 +22,6 @@ import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 
 import java.time.Duration;
-import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -111,10 +109,10 @@ public class DeploymentTester {
     }
 
     private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) {
-        List<JobType> triggerOrder = JobType.triggerOrder(SystemName.main, applicationPackage.deploymentSpec());
-        for (JobType job : triggerOrder) {
+        DeploymentOrder order = new DeploymentOrder(controller());
+        for (JobType job : order.jobsFrom(applicationPackage.deploymentSpec())) {
             boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false);
-            deployAndNotify(job, application, applicationPackage, !failJob);
+            deployAndNotify(application, applicationPackage, !failJob, job);
             if (failJob) {
                 break;
             }
@@ -163,20 +161,24 @@ public class DeploymentTester {
         job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion));
     }
 
-    public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) {
-        assertScheduledJob(application, job);
-        if (success) {
-            deploy(job, application, applicationPackage);
+    public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, JobType... jobs) {
+        assertScheduledJob(application, jobs);
+        for (JobType job : jobs) {
+            if (success) {
+                deploy(job, application, applicationPackage);
+            }
+            notifyJobCompletion(job, application, success);
         }
-        notifyJobCompletion(job, application, success);
     }
 
-    private void assertScheduledJob(Application application, JobType jobType) {
-        Optional<BuildService.BuildJob> job = findJob(application, jobType);
-        assertTrue(String.format("Job %s is scheduled for %s", jobType, application), job.isPresent());
+    private void assertScheduledJob(Application application, JobType... jobs) {
+        for (JobType job : jobs) {
+            Optional<BuildService.BuildJob> buildJob = findJob(application, job);
+            assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent());
+            assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId());
+            assertEquals(job.id(), buildJob.get().jobName());
+        }
         buildSystem().removeJobs(application.id());
-        assertEquals((long) application.deploymentJobs().projectId().get(), job.get().projectId());
-        assertEquals(jobType.id(), job.get().jobName());
     }
 
     private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 0f48afc0ca4..7ed0ad843cc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -6,13 +6,14 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
 import org.junit.Test;
 
 import java.time.Duration;
 
 import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -24,24 +25,42 @@ public class DeploymentTriggerTest {
     @Test
     public void testTriggerFailing() {
         DeploymentTester tester = new DeploymentTester();
-        Application app1 = tester.createAndDeploy("app1", 1, "default");
+        Application app = tester.createApplication("app1", "tenant1", 1, 1L);
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .upgradePolicy("default")
+                .environment(Environment.prod)
+                .region("us-west-1")
+                .build();
+
+        Version version = new Version(5, 1);
+        tester.updateVersionStatus(version);
+        tester.upgrader().maintain();
+
+        // Deploy completely once
+        tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, JobType.productionUsWest1);
 
-        Version version = new Version(5, 2);
-        tester.deploymentTrigger().triggerChange(app1.id(), new Change.VersionChange(version));
-        tester.completeUpgradeWithError(app1, version, "default", JobType.stagingTest);
+        // New version is released
+        version = new Version(5, 2);
+        tester.updateVersionStatus(version);
+        tester.upgrader().maintain();
+
+        tester.deployAndNotify(app, applicationPackage, false, JobType.systemTest);
         assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size());
 
         tester.buildSystem().takeJobsToRun();
-        assertEquals("Job removed", 0, tester.buildSystem().jobs().size());        
+        assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
         tester.clock().advance(Duration.ofHours(2));
-        tester.deploymentTrigger().triggerFailing(app1.id(), "unit test");
+        tester.failureRedeployer().maintain();
         assertEquals("Retried job", 1, tester.buildSystem().jobs().size());
-        assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName());
+        assertEquals(JobType.systemTest.id(), tester.buildSystem().jobs().get(0).jobName());
 
         tester.buildSystem().takeJobsToRun();
         assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
-        tester.clock().advance(Duration.ofHours(7));
-        tester.deploymentTrigger().triggerFailing(app1.id(), "unit test");
+        tester.clock().advance(Duration.ofHours(12).plus(Duration.ofSeconds(1)));
+        tester.failureRedeployer().maintain();
         assertEquals("Retried from the beginning", 1, tester.buildSystem().jobs().size());
         assertEquals(JobType.component.id(), tester.buildSystem().jobs().get(0).jobName());
     }
@@ -63,11 +82,11 @@ public class DeploymentTriggerTest {
         tester.notifyJobCompletion(JobType.component, application, true);
 
         // Application is deployed to all test environments and declared zones
-        tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.productionCorpUsEast1, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsWest1, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionCorpUsEast1);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsWest1);
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
     }
 
@@ -91,9 +110,9 @@ public class DeploymentTriggerTest {
         tester.notifyJobCompletion(JobType.component, application, true);
 
         // Test jobs pass
-        tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest);
         tester.clock().advance(Duration.ofSeconds(1)); // Make staging test sort as the last successful job
-        tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest);
         assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty());
 
         // 30 seconds pass, us-west-1 is triggered
@@ -121,7 +140,7 @@ public class DeploymentTriggerTest {
         // 3 minutes pass, us-central-1 is triggered
         tester.clock().advance(Duration.ofMinutes(3));
         tester.deploymentTrigger().triggerDelayed();
-        tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1);
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
 
         // Delayed trigger job runs again, with nothing to trigger
@@ -130,6 +149,77 @@ public class DeploymentTriggerTest {
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
     }
 
+    @Test
+    public void deploymentSpecWithParallelDeployments() {
+        DeploymentTester tester = new DeploymentTester();
+        Application application = tester.createApplication("app1", "tenant1", 1, 1L);
+
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .region("us-central-1")
+                .parallel("us-west-1", "us-east-3")
+                .region("eu-west-1")
+                .build();
+
+        // Component job finishes
+        tester.notifyJobCompletion(JobType.component, application, true);
+
+        // Test jobs pass
+        tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest);
+
+        // Deploys in first region
+        assertEquals(1, tester.buildSystem().jobs().size());
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1);
+
+        // Deploys in two regions in parallel
+        assertEquals(2, tester.buildSystem().jobs().size());
+        assertEquals(JobType.productionUsEast3.id(), tester.buildSystem().jobs().get(0).jobName());
+        assertEquals(JobType.productionUsWest1.id(), tester.buildSystem().jobs().get(1).jobName());
+        tester.buildSystem().takeJobsToRun();
+
+        tester.deploy(JobType.productionUsWest1, application, applicationPackage, false);
+        tester.notifyJobCompletion(JobType.productionUsWest1, application, true);
+        assertTrue("No more jobs triggered at this time", tester.buildSystem().jobs().isEmpty());
+
+        tester.deploy(JobType.productionUsEast3, application, applicationPackage, false);
+        tester.notifyJobCompletion(JobType.productionUsEast3, application, true);
+
+        // Last region completes
+        assertEquals(1, tester.buildSystem().jobs().size());
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionEuWest1);
+        assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
+    }
+
+    @Test
+    public void parallelDeploymentCompletesOutOfOrder() {
+        DeploymentTester tester = new DeploymentTester();
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .parallel("us-east-3", "us-west-1")
+                .build();
+
+        Application app = tester.createApplication("app1", "tenant1", 1, 11L);
+        tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
+
+        // Test environments pass
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+
+        // Parallel deployment
+        tester.deploy(DeploymentJobs.JobType.productionUsWest1, app, applicationPackage);
+        tester.deploy(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage);
+
+        // Last declared job completes first
+        tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsWest1, app, true);
+        assertTrue("Change is present as not all jobs are complete",
+                   tester.applications().require(app.id()).deploying().isPresent());
+
+        // All jobs complete
+        tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, true);
+        assertFalse("Change has been deployed",
+                    tester.applications().require(app.id()).deploying().isPresent());
+    }
 
     @Test
     public void testSuccessfulDeploymentApplicationPackageChanged() {
@@ -155,13 +245,13 @@ public class DeploymentTriggerTest {
         tester.notifyJobCompletion(JobType.component, application, true);
 
         // Application is deployed to all test environments and declared zones
-        tester.deployAndNotify(JobType.systemTest, application, newApplicationPackage, true);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.systemTest);
         tester.deploy(JobType.stagingTest, application, previousApplicationPackage, true);
-        tester.deployAndNotify(JobType.stagingTest, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionCorpUsEast1, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsCentral1, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsWest1, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionApNortheast1, application, newApplicationPackage, true);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.stagingTest);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionCorpUsEast1);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsCentral1);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsWest1);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionApNortheast1);
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
     }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
index f5a76f6446c..f8d09ac8b27 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
@@ -89,9 +89,9 @@ public class DeploymentIssueReporterTest {
         for (long i = 4; i <= 10; i++) {
             Application app = tester.createApplication("application" + i, "tenant" + i, 10 * i, i);
             tester.notifyJobCompletion(component, app, true);
-            tester.deployAndNotify(systemTest, app, applicationPackage, true);
-            tester.deployAndNotify(stagingTest, app, applicationPackage, true);
-            tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true);
+            tester.deployAndNotify(app, applicationPackage, true, systemTest);
+            tester.deployAndNotify(app, applicationPackage, true, stagingTest);
+            tester.deployAndNotify(app, applicationPackage, true, productionCorpUsEast1);
         }
 
         // Both the first tenants belong to the same JIRA queue. (Not sure if this is possible, but let's test it anyway.
@@ -111,17 +111,17 @@ public class DeploymentIssueReporterTest {
 
         // app1 and app3 has one failure each.
         tester.notifyJobCompletion(component, app1, true);
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, false);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, false, stagingTest);
 
         tester.notifyJobCompletion(component, app2, true);
-        tester.deployAndNotify(systemTest, app2, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app2, applicationPackage, true);
+        tester.deployAndNotify(app2, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app2, applicationPackage, true, stagingTest);
 
         tester.notifyJobCompletion(component, app3, true);
-        tester.deployAndNotify(systemTest, app3, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app3, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false);
+        tester.deployAndNotify(app3, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app3, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1);
 
         reporter.maintain();
         reporter.maintain();
@@ -157,7 +157,7 @@ public class DeploymentIssueReporterTest {
 
         // Some time passes; tenant1 leaves her issue unattended, while tenant3 starts work and updates the issue.
         // app2 also has an intermittent failure; see that we detect this as a Vespa problem, and file an issue to ourselves.
-        tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, false);
+        tester.deployAndNotify(app2, applicationPackage, false, productionCorpUsEast1);
         tester.clock().advance(maxInactivityAge.plus(maxFailureAge));
         issues.comment(openIssuesFor(app3).get(0).id(), "We are trying to fix it!");
 
@@ -177,8 +177,8 @@ public class DeploymentIssueReporterTest {
 
 
         // app3 fixes its problem, but the ticket is left open; see the resolved ticket is not escalated when another escalation period has passed.
-        tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, true);
+        tester.deployAndNotify(app2, applicationPackage, true, productionCorpUsEast1);
+        tester.deployAndNotify(app3, applicationPackage, true, productionCorpUsEast1);
         tester.clock().advance(maxInactivityAge.plus(Duration.ofDays(1)));
 
         reporter.maintain();
@@ -190,9 +190,9 @@ public class DeploymentIssueReporterTest {
         // app1 still does nothing with their issue; see the terminal user gets it in the end.
         // app3 now has a new failure past max failure age; see that a new issue is filed.
         tester.notifyJobCompletion(component, app3, true);
-        tester.deployAndNotify(systemTest, app3, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app3, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false);
+        tester.deployAndNotify(app3, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app3, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1);
         tester.clock().advance(maxInactivityAge.plus(maxFailureAge));
 
         reporter.maintain();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
index 052ca87f791..b5ee0469e9f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -41,9 +41,9 @@ public class FailureRedeployerTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
 
         // New version is released
         version = Version.fromString("5.1");
@@ -52,12 +52,12 @@ public class FailureRedeployerTest {
         tester.upgrader().maintain();
 
         // Test environments pass
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
 
         // Production job fails and is retried
         tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
         assertEquals("Production job is retried", 1, tester.buildSystem().jobs().size());
         assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
 
@@ -75,11 +75,11 @@ public class FailureRedeployerTest {
                 .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id())));
 
         // Test environments pass
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
 
         // Production job fails again and exhausts all immediate retries
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
         tester.buildSystem().takeJobsToRun();
         tester.clock().advance(Duration.ofMinutes(10));
         tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, false);
@@ -92,7 +92,7 @@ public class FailureRedeployerTest {
         assertEquals("Job is retried", 1, tester.buildSystem().jobs().size());
 
         // Production job finally succeeds
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
         assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
         assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures());
     }
@@ -108,7 +108,7 @@ public class FailureRedeployerTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
 
         // staging-test starts, but does not complete
         assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName());
@@ -139,9 +139,9 @@ public class FailureRedeployerTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
 
         // New version is released
         version = Version.fromString("5.1");
@@ -151,7 +151,7 @@ public class FailureRedeployerTest {
         assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
 
         // system-test fails and exhausts all immediate retries
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.systemTest);
         tester.buildSystem().takeJobsToRun();
         tester.clock().advance(Duration.ofMinutes(10));
         tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, app, false);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index a832a591217..3244307e91c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -84,7 +84,7 @@ public class MetricsReporterTest {
 
         // 1 app fails system-test
         tester.notifyJobCompletion(component, app4, true);
-        tester.deployAndNotify(systemTest, app4, applicationPackage, false);
+        tester.deployAndNotify(app4, applicationPackage, false, systemTest);
 
         metricsReporter.maintain();
         assertEquals(25.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index e5afcec87ad..e047a288fb9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -224,9 +224,9 @@ public class UpgraderTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
 
         tester.upgrader().maintain();
         assertEquals("Application is on expected version: Nothing to do", 0,
@@ -239,10 +239,10 @@ public class UpgraderTest {
         tester.upgrader().maintain();
 
         // system-test completes successfully
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
 
         // staging-test fails multiple times, exhausts retries and failure is recorded
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.stagingTest);
         tester.buildSystem().takeJobsToRun();
         tester.clock().advance(Duration.ofMinutes(10));
         tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, app, false);
@@ -282,17 +282,17 @@ public class UpgraderTest {
         // Application is on 5.0
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionCorpUsEast1);
 
         // Canary in prod.corp-us-east-1 is upgraded to controller version
         tester.upgrader().maintain();
         assertEquals("Upgrade started", 1, tester.buildSystem().jobs().size());
         assertEquals(Vtag.currentVersion, ((Change.VersionChange) tester.application(app.id()).deploying().get()).version());
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionCorpUsEast1);
 
         // System is upgraded to newer version, no upgrade triggered for canary as version is lower than controller
         version = Version.fromString("5.1");