Trigger parallel deployments

11 files changed, 260 insertions, 169 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index 61231404a94..19d60c18998 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.application;
 
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.RegionName;
@@ -16,11 +15,9 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.stream.Collectors;
 
 /**
  * Information about which deployment jobs an application should run and their current status.
@@ -252,14 +249,6 @@ public class DeploymentJobs {
             return from(system, new com.yahoo.config.provision.Zone(environment, region));
         }
 
-        /** Returns the trigger order to use according to deployment spec */
-        public static List<JobType> triggerOrder(SystemName system, DeploymentSpec deploymentSpec) {
-            return deploymentSpec.zones().stream()
-                    .map(declaredZone -> JobType.from(system, declaredZone.environment(),
-                                                      declaredZone.region().orElse(null)))
-                    .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
-        }
-
         private static Zone zone(SystemName system, String environment, String region) {
             return new Zone(system, Environment.from(environment), RegionName.from(region));
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
index 113fe4ce1f2..afa9c219048 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
@@ -3,19 +3,24 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
 import java.util.Optional;
 import java.util.logging.Logger;
+import java.util.stream.Collectors;
+
+import static java.util.stream.Collectors.collectingAndThen;
 
 /**
- * This class determines order of deployments according to an application's deployment spec
+ * This class determines the order of deployments according to an application's deployment spec.
  *
  * @author mpolden
  */
@@ -31,68 +36,124 @@ public class DeploymentOrder {
         this.clock = controller.clock();
     }
 
-    /** Returns the next job(s) to trigger after the given job, or empty if none should be triggered */
-    public List<DeploymentJobs.JobType> nextAfter(DeploymentJobs.JobType jobType, Application application) {
+    /** Returns a list of jobs to trigger after the given job */
+    public List<JobType> nextAfter(JobType job, Application application) {
         // Always trigger system test after component as deployment spec might not be available yet (e.g. if this is a
         // new application with no previous deployments)
-        if (jobType == DeploymentJobs.JobType.component) {
-            return Collections.singletonList(DeploymentJobs.JobType.systemTest);
+        if (job == JobType.component) {
+            return Collections.singletonList(JobType.systemTest);
         }
 
-        // At this point we've at least deployed to system test, so deployment spec should be available
-        List<DeploymentSpec.DeclaredZone> zones = application.deploymentSpec().zones();
-        Optional<DeploymentSpec.DeclaredZone> zoneForJob = zoneForJob(application, jobType);
-        if (!zoneForJob.isPresent()) {
+        // At this point we have deployed to system test, so deployment spec is available
+        List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application);
+        Optional<DeploymentSpec.Step> currentStep = fromJob(job, application);
+        if (!currentStep.isPresent()) {
             return Collections.emptyList();
         }
-        int zoneIndex = application.deploymentSpec().zones().indexOf(zoneForJob.get());
 
-        // This is last zone
-        if (zoneIndex == zones.size() - 1) {
+        // If this is the last deployment step there's nothing more to trigger
+        int currentIndex = deploymentSteps.indexOf(currentStep.get());
+        if (currentIndex == deploymentSteps.size() - 1) {
             return Collections.emptyList();
         }
 
-        // Skip next job if delay has not passed yet
-        Duration delay = delayAfter(application, zoneForJob.get());
-        Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType))
-                .flatMap(JobStatus::lastSuccess)
-                .map(JobStatus.JobRun::at);
-        if (lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant())) {
-            log.info(String.format("Delaying next job after %s of %s by %s", jobType, application, delay));
+        // Postpone next job if delay has not passed yet
+        Duration delay = delayAfter(currentStep.get(), application);
+        if (postponeDeployment(delay, job, application)) {
+            log.info(String.format("Delaying next job after %s of %s by %s", job, application, delay));
             return Collections.emptyList();
         }
 
-        DeploymentSpec.DeclaredZone nextZone = application.deploymentSpec().zones().get(zoneIndex + 1);
-        return Collections.singletonList(
-                DeploymentJobs.JobType.from(controller.system(), nextZone.environment(), nextZone.region().orElse(null))
-        );
+        DeploymentSpec.Step nextStep = deploymentSteps.get(currentIndex + 1);
+        if (nextStep instanceof DeploymentSpec.DeclaredZone) {
+            return Collections.singletonList(toJob((DeploymentSpec.DeclaredZone) nextStep));
+        } else if (nextStep instanceof DeploymentSpec.ParallelZones) {
+            return ((DeploymentSpec.ParallelZones) nextStep).zones().stream()
+                    .map(this::toJob)
+                    .collect(collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
+        } else {
+            throw new IllegalStateException("Unexpected step type: " + nextStep.getClass());
+        }
+    }
+
+    /** Returns whether the given job is first in a deployment */
+    public boolean isFirst(JobType job) {
+        return job == JobType.component;
+    }
+
+    /** Returns whether the given job is last in a deployment */
+    public boolean isLast(JobType job, Application application) {
+        List<DeploymentSpec.Step> deploymentSteps = deploymentSteps(application);
+        if (deploymentSteps.isEmpty()) { // Deployment spec not yet available
+            return false;
+        }
+        DeploymentSpec.Step lastStep = deploymentSteps.get(deploymentSteps.size() - 1);
+        return fromJob(job, application).get().equals(lastStep);
+    }
+
+    /** Returns jobs for given deployment spec, in the order they are declared */
+    public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) {
+        if (deploymentSpec.steps().isEmpty()) {
+            return Arrays.asList(JobType.systemTest, JobType.stagingTest);
+        }
+        List<JobType> jobs = new ArrayList<>();
+        for (DeploymentSpec.Step step : deploymentSpec.steps()) {
+            if (step instanceof DeploymentSpec.DeclaredZone) {
+                jobs.add(toJob((DeploymentSpec.DeclaredZone) step));
+            } else if (step instanceof DeploymentSpec.ParallelZones) {
+                ((DeploymentSpec.ParallelZones) step).zones().forEach(zone -> jobs.add(toJob(zone)));
+            }
+        }
+        return Collections.unmodifiableList(jobs);
+    }
+
+    /** Resolve deployment step from job */
+    private Optional<DeploymentSpec.Step> fromJob(JobType job, Application application) {
+        for (DeploymentSpec.Step step : application.deploymentSpec().steps()) {
+            if (step.deploysTo(job.environment(), job.isProduction() ? job.region(controller.system()) : Optional.empty())) {
+                return Optional.of(step);
+            }
+        }
+        return Optional.empty();
+    }
+
+    /** Resolve job from deployment step */
+    private JobType toJob(DeploymentSpec.DeclaredZone zone) {
+        return JobType.from(controller.system(), zone.environment(), zone.region().orElse(null));
+    }
+
+    /** Returns whether deployment should be postponed according to delay */
+    private boolean postponeDeployment(Duration delay, JobType job, Application application) {
+        Optional<Instant> lastSuccess = Optional.ofNullable(application.deploymentJobs().jobStatus().get(job))
+                .flatMap(JobStatus::lastSuccess)
+                .map(JobStatus.JobRun::at);
+        return lastSuccess.isPresent() && lastSuccess.get().plus(delay).isAfter(clock.instant());
     }
 
-    private Duration delayAfter(Application application, DeploymentSpec.DeclaredZone zone) {
-        int stepIndex = application.deploymentSpec().steps().indexOf(zone);
+    /** Find all steps that deploy to one or more zones */
+    private static List<DeploymentSpec.Step> deploymentSteps(Application application) {
+        return application.deploymentSpec().steps().stream()
+                .filter(step -> step instanceof DeploymentSpec.DeclaredZone ||
+                                step instanceof DeploymentSpec.ParallelZones)
+                .collect(Collectors.toList());
+    }
+
+    /** Determines the delay that should pass after the given step */
+    private static Duration delayAfter(DeploymentSpec.Step step, Application application) {
+        int stepIndex = application.deploymentSpec().steps().indexOf(step);
         if (stepIndex == -1 || stepIndex == application.deploymentSpec().steps().size() - 1) {
             return Duration.ZERO;
         }
         Duration totalDelay = Duration.ZERO;
         List<DeploymentSpec.Step> remainingSteps = application.deploymentSpec().steps()
                 .subList(stepIndex + 1, application.deploymentSpec().steps().size());
-        for (DeploymentSpec.Step step : remainingSteps) {
-            if (!(step instanceof DeploymentSpec.Delay)) {
+        for (DeploymentSpec.Step s : remainingSteps) {
+            if (!(s instanceof DeploymentSpec.Delay)) {
                 break;
             }
-            totalDelay = totalDelay.plus(((DeploymentSpec.Delay) step).duration());
+            totalDelay = totalDelay.plus(((DeploymentSpec.Delay) s).duration());
         }
         return totalDelay;
     }
 
-    private Optional<DeploymentSpec.DeclaredZone> zoneForJob(Application application, DeploymentJobs.JobType jobType) {
-        return application.deploymentSpec()
-                .zones()
-                .stream()
-                .filter(zone -> zone.deploysTo(
-                        jobType.environment(),
-                        jobType.isProduction() ? jobType.region(controller.system()) : Optional.empty()))
-                .findFirst();
-    }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 1ccbd70d4d2..12c8c508869 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -64,7 +64,7 @@ public class DeploymentTrigger {
             application = application.withJobCompletion(report, clock.instant(), controller);
             
             // Handle successful first and last job
-            if (isFirstJob(report.jobType()) && report.success()) { // the first job tells us that a change occurred
+            if (order.isFirst(report.jobType()) && report.success()) { // the first job tells us that a change occurred
                 if (application.deploying().isPresent() && ! application.deploymentJobs().hasFailures()) { // postpone until the current deployment is done
                     applications().store(application.withOutstandingChange(true), lock);
                     return;
@@ -72,7 +72,7 @@ public class DeploymentTrigger {
                 else { // start a new change deployment
                     application = application.withDeploying(Optional.of(Change.ApplicationChange.unknown()));
                 }
-            } else if (isLastJob(report.jobType(), application) && report.success()) {
+            } else if (order.isLast(report.jobType(), application) && report.success()) {
                 application = application.withDeploying(Optional.empty());
             }
 
@@ -101,7 +101,7 @@ public class DeploymentTrigger {
                 applications().store(application, lock);
             } else {
                 // retry the failed job (with backoff)
-                for (JobType jobType : JobType.triggerOrder(controller.system(), application.deploymentSpec())) { // retry the *first* failing job
+                for (JobType jobType : order.jobsFrom(application.deploymentSpec())) { // retry the *first* failing job
                     JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
                     if (isFailing(jobStatus)) {
                         if (shouldRetryNow(jobStatus)) {
@@ -178,15 +178,6 @@ public class DeploymentTrigger {
     //--- End of methods which triggers deployment jobs ----------------------------
 
     private ApplicationController applications() { return controller.applications(); }
-
-    private boolean isFirstJob(JobType jobType) {
-        return jobType == JobType.component;
-    }
-
-    private boolean isLastJob(JobType jobType, Application application) {
-        List<JobType> triggerOrder = JobType.triggerOrder(controller.system(), application.deploymentSpec());
-        return triggerOrder.isEmpty() || jobType.equals(triggerOrder.get(triggerOrder.size() - 1));
-    }
     
     private boolean isFailing(JobStatus jobStatusOrNull) {
         return jobStatusOrNull != null && !jobStatusOrNull.isSuccess();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 3dddfaf58a1..29b34747573 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -104,8 +104,8 @@ public class ControllerTest {
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
         assertFalse("Revision is currently not known",
                     ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision().isPresent());
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
         assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
 
         Optional<ApplicationRevision> revision = ((Change.ApplicationChange)tester.controller().applications().require(app1.id()).deploying().get()).revision();
@@ -120,7 +120,7 @@ public class ControllerTest {
         tester.clock().advance(Duration.ofSeconds(1));
 
         // production job (failing)
-        tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, false);
+        tester.deployAndNotify(app1, applicationPackage, false, productionCorpUsEast1);
         assertEquals(4, applications.require(app1.id()).deploymentJobs().jobStatus().size());
 
         JobStatus expectedJobStatus = JobStatus.initial(productionCorpUsEast1)
@@ -144,14 +144,14 @@ public class ControllerTest {
 
         // system and staging test job - succeeding
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
         assertStatus(JobStatus.initial(systemTest)
                               .withTriggering(version1, revision, tester.clock().instant())
                               .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller()), app1.id(), tester.controller());
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
 
         // production job succeeding now
-        tester.deployAndNotify(productionCorpUsEast1, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, productionCorpUsEast1);
         expectedJobStatus = expectedJobStatus
                 .withTriggering(version1, revision, tester.clock().instant())
                 .withCompletion(Optional.empty(), tester.clock().instant(), tester.controller());
@@ -161,7 +161,7 @@ public class ControllerTest {
         assertStatus(JobStatus.initial(productionUsEast3)
                               .withTriggering( version1, revision, tester.clock().instant()), 
                      app1.id(), tester.controller());
-        tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
 
         assertEquals(5, applications.get(app1.id()).get().deploymentJobs().jobStatus().size());
         
@@ -189,7 +189,7 @@ public class ControllerTest {
                 .environment(Environment.prod)
                 .region("us-east-3")
                 .build();
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
         assertNull("Zone was removed",
                    applications.require(app1.id()).deployments().get(productionCorpUsEast1.zone(SystemName.main).get()));
         assertNull("Deployment job was removed", applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1));
@@ -211,9 +211,9 @@ public class ControllerTest {
 
         // First deployment: An application change
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
 
         app1 = applications.require(app1.id());
         assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get());
@@ -234,9 +234,9 @@ public class ControllerTest {
                 .region("us-east-3")
                 .build();
         applications.notifyJobCompletion(mockReport(app1, component, true, false));
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
 
         app1 = applications.require(app1.id());
         assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
@@ -248,7 +248,7 @@ public class ControllerTest {
                 .region("us-west-1")
                 .region("us-east-3")
                 .build();
-        tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
         app1 = applications.require(app1.id());
         assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
         assertEquals(systemVersion, tester.configServerClientMock().lastPrepareVersion.get());
@@ -256,10 +256,10 @@ public class ControllerTest {
         // Version upgrade changes system version
         Change.VersionChange change = new Change.VersionChange(newSystemVersion);
         applications.deploymentTrigger().triggerChange(app1.id(), change);
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsWest1, app1, applicationPackage, true);
-        tester.deployAndNotify(productionUsEast3, app1, applicationPackage, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
+        tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
 
         app1 = applications.require(app1.id());
         assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get());
@@ -326,37 +326,37 @@ public class ControllerTest {
         // Initial failure
         Instant initialFailure = tester.clock().instant();
         tester.notifyJobCompletion(component, app, true);
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at initial failure", 
                      initialFailure, firstFailing(app, tester).get().at());
 
         // Failure again -- failingSince should remain the same
         tester.clock().advance(Duration.ofMillis(1000));
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at second consecutive failure", 
                      initialFailure, firstFailing(app, tester).get().at());
 
         // Success resets failingSince
         tester.clock().advance(Duration.ofMillis(1000));
-        tester.deployAndNotify(systemTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, systemTest);
         assertFalse(firstFailing(app, tester).isPresent());
 
         // Complete deployment
-        tester.deployAndNotify(stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, productionCorpUsEast1);
         
         // Two repeated failures again.
         // Initial failure
         tester.clock().advance(Duration.ofMillis(1000));
         initialFailure = tester.clock().instant();
         tester.notifyJobCompletion(component, app, true);
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at initial failure", 
                      initialFailure, firstFailing(app, tester).get().at());
 
         // Failure again -- failingSince should remain the same
         tester.clock().advance(Duration.ofMillis(1000));
-        tester.deployAndNotify(systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, systemTest);
         assertEquals("Failure age is right at second consecutive failure", 
                      initialFailure, firstFailing(app, tester).get().at());
     }
@@ -435,11 +435,11 @@ public class ControllerTest {
 
         // foo: passes system test
         tester.notifyJobCompletion(component, foo, true);
-        tester.deployAndNotify(systemTest, foo, applicationPackage, true);
+        tester.deployAndNotify(foo, applicationPackage, true, systemTest);
 
         // bar: passes system test
         tester.notifyJobCompletion(component, bar, true);
-        tester.deployAndNotify(systemTest, bar, applicationPackage, true);
+        tester.deployAndNotify(bar, applicationPackage, true, systemTest);
 
         // foo and bar: staging test jobs queued
         assertEquals(2, buildSystem.jobs().size());
@@ -455,14 +455,14 @@ public class ControllerTest {
         }
 
         // bar: Completes deployment
-        tester.deployAndNotify(stagingTest, bar, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, bar, applicationPackage, true);
+        tester.deployAndNotify(bar, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(bar, applicationPackage, true, productionCorpUsEast1);
 
         // foo: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by
         // out of capacity retry mechanism
         tester.clock().advance(Duration.ofMinutes(15));
         tester.notifyJobCompletion(component, foo, true);
-        tester.deployAndNotify(systemTest, foo, applicationPackage, true);
+        tester.deployAndNotify(foo, applicationPackage, true, systemTest);
         tester.deploy(stagingTest, foo, applicationPackage);
         assertEquals(1, buildSystem.takeJobsToRun().size());
         tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity));
@@ -470,7 +470,7 @@ public class ControllerTest {
 
         // bar: New change triggers another staging-test job
         tester.notifyJobCompletion(component, bar, true);
-        tester.deployAndNotify(systemTest, bar, applicationPackage, true);
+        tester.deployAndNotify(bar, applicationPackage, true, systemTest);
         assertEquals(1, buildSystem.jobs().size());
 
         // foo: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index aa115421f6a..23451c60f08 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -13,6 +13,7 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.ZoneId;
 import java.time.format.DateTimeFormatter;
+import java.util.Arrays;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
 
@@ -45,6 +46,13 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
+    public ApplicationPackageBuilder parallel(String... regionName) {
+        environmentBody.append("    <parallel>\n");
+        Arrays.stream(regionName).forEach(this::region);
+        environmentBody.append("    </parallel>\n");
+        return this;
+    }
+
     public ApplicationPackageBuilder delay(Duration delay) {
         environmentBody.append("    <delay seconds='");
         environmentBody.append(delay.getSeconds());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 0e816be864d..e04ff8576e3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.SystemName;
 import com.yahoo.test.ManualClock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
@@ -23,7 +22,6 @@ import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
 import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 
 import java.time.Duration;
-import java.util.List;
 import java.util.Optional;
 import java.util.UUID;
 
@@ -111,10 +109,10 @@ public class DeploymentTester {
     }
 
     private void completeDeployment(Application application, ApplicationPackage applicationPackage, Optional<JobType> failOnJob) {
-        List<JobType> triggerOrder = JobType.triggerOrder(SystemName.main, applicationPackage.deploymentSpec());
-        for (JobType job : triggerOrder) {
+        DeploymentOrder order = new DeploymentOrder(controller());
+        for (JobType job : order.jobsFrom(applicationPackage.deploymentSpec())) {
             boolean failJob = failOnJob.map(j -> j.equals(job)).orElse(false);
-            deployAndNotify(job, application, applicationPackage, !failJob);
+            deployAndNotify(application, applicationPackage, !failJob, job);
             if (failJob) {
                 break;
             }
@@ -163,20 +161,25 @@ public class DeploymentTester {
         job.zone(controller().system()).ifPresent(zone -> tester.deploy(application, zone, applicationPackage, deployCurrentVersion));
     }
 
-    public void deployAndNotify(JobType job, Application application, ApplicationPackage applicationPackage, boolean success) {
-        assertScheduledJob(application, job);
-        if (success) {
-            deploy(job, application, applicationPackage);
+    public void deployAndNotify(Application application, ApplicationPackage applicationPackage, boolean success, JobType... jobs) {
+        assertScheduledJob(application, jobs);
+        for (JobType job : jobs) {
+            if (success) {
+                deploy(job, application, applicationPackage);
+            }
+            notifyJobCompletion(job, application, success);
         }
-        notifyJobCompletion(job, application, success);
     }
 
-    private void assertScheduledJob(Application application, JobType jobType) {
-        Optional<BuildService.BuildJob> job = findJob(application, jobType);
-        assertTrue(String.format("Job %s is scheduled for %s", jobType, application), job.isPresent());
+    private void assertScheduledJob(Application application, JobType... jobs) {
+        for (JobType job : jobs) {
+            Optional<BuildService.BuildJob> buildJob = findJob(application, job);
+            assertTrue(String.format("Job %s is scheduled for %s", job, application), buildJob.isPresent());
+            assertEquals((long) application.deploymentJobs().projectId().get(), buildJob.get().projectId());
+            assertEquals(job.id(), buildJob.get().jobName());
+        }
         buildSystem().removeJobs(application.id());
-        assertEquals((long) application.deploymentJobs().projectId().get(), job.get().projectId());
-        assertEquals(jobType.id(), job.get().jobName());
+
     }
 
     private Optional<BuildService.BuildJob> findJob(Application application, JobType jobType) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 0f48afc0ca4..7b658dac6f9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -24,11 +24,18 @@ public class DeploymentTriggerTest {
     @Test
     public void testTriggerFailing() {
         DeploymentTester tester = new DeploymentTester();
-        Application app1 = tester.createAndDeploy("app1", 1, "default");
+        Application app1 = tester.createApplication("app1", "tenant1", 1, 1L);
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .upgradePolicy("default")
+                .environment(Environment.prod)
+                .region("us-west-1")
+                .build();
+        app1 = app1.with(app1.deploymentJobs().asSelfTriggering(false));
+        tester.applications().store(app1, tester.applications().lock(app1.id()));
 
         Version version = new Version(5, 2);
         tester.deploymentTrigger().triggerChange(app1.id(), new Change.VersionChange(version));
-        tester.completeUpgradeWithError(app1, version, "default", JobType.stagingTest);
+        tester.deployAndNotify(app1, applicationPackage, false, JobType.systemTest);
         assertEquals("Retried immediately", 1, tester.buildSystem().jobs().size());
 
         tester.buildSystem().takeJobsToRun();
@@ -36,7 +43,7 @@ public class DeploymentTriggerTest {
         tester.clock().advance(Duration.ofHours(2));
         tester.deploymentTrigger().triggerFailing(app1.id(), "unit test");
         assertEquals("Retried job", 1, tester.buildSystem().jobs().size());
-        assertEquals(JobType.stagingTest.id(), tester.buildSystem().jobs().get(0).jobName());
+        assertEquals(JobType.systemTest.id(), tester.buildSystem().jobs().get(0).jobName());
 
         tester.buildSystem().takeJobsToRun();
         assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
@@ -63,11 +70,11 @@ public class DeploymentTriggerTest {
         tester.notifyJobCompletion(JobType.component, application, true);
 
         // Application is deployed to all test environments and declared zones
-        tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.productionCorpUsEast1, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsWest1, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionCorpUsEast1);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsWest1);
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
     }
 
@@ -91,9 +98,9 @@ public class DeploymentTriggerTest {
         tester.notifyJobCompletion(JobType.component, application, true);
 
         // Test jobs pass
-        tester.deployAndNotify(JobType.systemTest, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest);
         tester.clock().advance(Duration.ofSeconds(1)); // Make staging test sort as the last successful job
-        tester.deployAndNotify(JobType.stagingTest, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest);
         assertTrue("No more jobs triggered at this time", buildSystem.jobs().isEmpty());
 
         // 30 seconds pass, us-west-1 is triggered
@@ -121,7 +128,7 @@ public class DeploymentTriggerTest {
         // 3 minutes pass, us-central-1 is triggered
         tester.clock().advance(Duration.ofMinutes(3));
         tester.deploymentTrigger().triggerDelayed();
-        tester.deployAndNotify(JobType.productionUsCentral1, application, applicationPackage, true);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1);
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
 
         // Delayed trigger job runs again, with nothing to trigger
@@ -130,6 +137,38 @@ public class DeploymentTriggerTest {
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
     }
 
+    @Test
+    public void deploymentSpecWithParallelDeployments() {
+        DeploymentTester tester = new DeploymentTester();
+        Application application = tester.createApplication("app1", "tenant1", 1, 1L);
+
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .region("us-central-1")
+                .parallel("us-west-1", "us-east-3")
+                .build();
+
+        // Component job finishes
+        tester.notifyJobCompletion(JobType.component, application, true);
+
+        // Test jobs pass
+        tester.deployAndNotify(application, applicationPackage, true, JobType.systemTest);
+        tester.deployAndNotify(application, applicationPackage, true, JobType.stagingTest);
+
+        // Deploys in first region
+        assertEquals(1, tester.buildSystem().jobs().size());
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsCentral1);
+
+        // The two next regions are triggered in parallel
+        assertEquals(2, tester.buildSystem().jobs().size());
+        assertEquals(JobType.productionUsEast3.id(), tester.buildSystem().jobs().get(0).jobName());
+        assertEquals(JobType.productionUsWest1.id(), tester.buildSystem().jobs().get(1).jobName());
+
+        // Deployment completes
+        tester.deployAndNotify(application, applicationPackage, true, JobType.productionUsWest1,
+                               JobType.productionUsEast3);
+        assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
+    }
 
     @Test
     public void testSuccessfulDeploymentApplicationPackageChanged() {
@@ -155,13 +194,13 @@ public class DeploymentTriggerTest {
         tester.notifyJobCompletion(JobType.component, application, true);
 
         // Application is deployed to all test environments and declared zones
-        tester.deployAndNotify(JobType.systemTest, application, newApplicationPackage, true);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.systemTest);
         tester.deploy(JobType.stagingTest, application, previousApplicationPackage, true);
-        tester.deployAndNotify(JobType.stagingTest, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionCorpUsEast1, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsCentral1, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionUsWest1, application, newApplicationPackage, true);
-        tester.deployAndNotify(JobType.productionApNortheast1, application, newApplicationPackage, true);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.stagingTest);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionCorpUsEast1);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsCentral1);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionUsWest1);
+        tester.deployAndNotify(application, newApplicationPackage, true, JobType.productionApNortheast1);
         assertTrue("All jobs consumed", buildSystem.jobs().isEmpty());
     }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
index f5a76f6446c..f8d09ac8b27 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentIssueReporterTest.java
@@ -89,9 +89,9 @@ public class DeploymentIssueReporterTest {
         for (long i = 4; i <= 10; i++) {
             Application app = tester.createApplication("application" + i, "tenant" + i, 10 * i, i);
             tester.notifyJobCompletion(component, app, true);
-            tester.deployAndNotify(systemTest, app, applicationPackage, true);
-            tester.deployAndNotify(stagingTest, app, applicationPackage, true);
-            tester.deployAndNotify(productionCorpUsEast1, app, applicationPackage, true);
+            tester.deployAndNotify(app, applicationPackage, true, systemTest);
+            tester.deployAndNotify(app, applicationPackage, true, stagingTest);
+            tester.deployAndNotify(app, applicationPackage, true, productionCorpUsEast1);
         }
 
         // Both the first tenants belong to the same JIRA queue. (Not sure if this is possible, but let's test it anyway.
@@ -111,17 +111,17 @@ public class DeploymentIssueReporterTest {
 
         // app1 and app3 has one failure each.
         tester.notifyJobCompletion(component, app1, true);
-        tester.deployAndNotify(systemTest, app1, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app1, applicationPackage, false);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app1, applicationPackage, false, stagingTest);
 
         tester.notifyJobCompletion(component, app2, true);
-        tester.deployAndNotify(systemTest, app2, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app2, applicationPackage, true);
+        tester.deployAndNotify(app2, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app2, applicationPackage, true, stagingTest);
 
         tester.notifyJobCompletion(component, app3, true);
-        tester.deployAndNotify(systemTest, app3, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app3, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false);
+        tester.deployAndNotify(app3, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app3, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1);
 
         reporter.maintain();
         reporter.maintain();
@@ -157,7 +157,7 @@ public class DeploymentIssueReporterTest {
 
         // Some time passes; tenant1 leaves her issue unattended, while tenant3 starts work and updates the issue.
         // app2 also has an intermittent failure; see that we detect this as a Vespa problem, and file an issue to ourselves.
-        tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, false);
+        tester.deployAndNotify(app2, applicationPackage, false, productionCorpUsEast1);
         tester.clock().advance(maxInactivityAge.plus(maxFailureAge));
         issues.comment(openIssuesFor(app3).get(0).id(), "We are trying to fix it!");
 
@@ -177,8 +177,8 @@ public class DeploymentIssueReporterTest {
 
 
         // app3 fixes its problem, but the ticket is left open; see the resolved ticket is not escalated when another escalation period has passed.
-        tester.deployAndNotify(productionCorpUsEast1, app2, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, true);
+        tester.deployAndNotify(app2, applicationPackage, true, productionCorpUsEast1);
+        tester.deployAndNotify(app3, applicationPackage, true, productionCorpUsEast1);
         tester.clock().advance(maxInactivityAge.plus(Duration.ofDays(1)));
 
         reporter.maintain();
@@ -190,9 +190,9 @@ public class DeploymentIssueReporterTest {
         // app1 still does nothing with their issue; see the terminal user gets it in the end.
         // app3 now has a new failure past max failure age; see that a new issue is filed.
         tester.notifyJobCompletion(component, app3, true);
-        tester.deployAndNotify(systemTest, app3, applicationPackage, true);
-        tester.deployAndNotify(stagingTest, app3, applicationPackage, true);
-        tester.deployAndNotify(productionCorpUsEast1, app3, applicationPackage, false);
+        tester.deployAndNotify(app3, applicationPackage, true, systemTest);
+        tester.deployAndNotify(app3, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app3, applicationPackage, false, productionCorpUsEast1);
         tester.clock().advance(maxInactivityAge.plus(maxFailureAge));
 
         reporter.maintain();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
index 052ca87f791..b5ee0469e9f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/FailureRedeployerTest.java
@@ -41,9 +41,9 @@ public class FailureRedeployerTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
 
         // New version is released
         version = Version.fromString("5.1");
@@ -52,12 +52,12 @@ public class FailureRedeployerTest {
         tester.upgrader().maintain();
 
         // Test environments pass
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
 
         // Production job fails and is retried
         tester.clock().advance(Duration.ofSeconds(1)); // Advance time so that we can detect jobs in progress
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
         assertEquals("Production job is retried", 1, tester.buildSystem().jobs().size());
         assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
 
@@ -75,11 +75,11 @@ public class FailureRedeployerTest {
                 .anyMatch(j -> j.jobName().equals(DeploymentJobs.JobType.productionUsEast3.id())));
 
         // Test environments pass
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
 
         // Production job fails again and exhausts all immediate retries
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.productionUsEast3);
         tester.buildSystem().takeJobsToRun();
         tester.clock().advance(Duration.ofMinutes(10));
         tester.notifyJobCompletion(DeploymentJobs.JobType.productionUsEast3, app, false);
@@ -92,7 +92,7 @@ public class FailureRedeployerTest {
         assertEquals("Job is retried", 1, tester.buildSystem().jobs().size());
 
         // Production job finally succeeds
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
         assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
         assertFalse("No failures", tester.application(app.id()).deploymentJobs().hasFailures());
     }
@@ -108,7 +108,7 @@ public class FailureRedeployerTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
 
         // staging-test starts, but does not complete
         assertEquals(DeploymentJobs.JobType.stagingTest.id(), tester.buildSystem().takeJobsToRun().get(0).jobName());
@@ -139,9 +139,9 @@ public class FailureRedeployerTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
 
         // New version is released
         version = Version.fromString("5.1");
@@ -151,7 +151,7 @@ public class FailureRedeployerTest {
         assertEquals("Application has pending upgrade to " + version, version, tester.versionChange(app.id()).get().version());
 
         // system-test fails and exhausts all immediate retries
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.systemTest);
         tester.buildSystem().takeJobsToRun();
         tester.clock().advance(Duration.ofMinutes(10));
         tester.notifyJobCompletion(DeploymentJobs.JobType.systemTest, app, false);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
index a832a591217..3244307e91c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java
@@ -84,7 +84,7 @@ public class MetricsReporterTest {
 
         // 1 app fails system-test
         tester.notifyJobCompletion(component, app4, true);
-        tester.deployAndNotify(systemTest, app4, applicationPackage, false);
+        tester.deployAndNotify(app4, applicationPackage, false, systemTest);
 
         metricsReporter.maintain();
         assertEquals(25.0, metricsMock.getMetric(MetricsReporter.deploymentFailMetric));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index e5afcec87ad..e047a288fb9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -224,9 +224,9 @@ public class UpgraderTest {
 
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionUsEast3, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionUsEast3);
 
         tester.upgrader().maintain();
         assertEquals("Application is on expected version: Nothing to do", 0,
@@ -239,10 +239,10 @@ public class UpgraderTest {
         tester.upgrader().maintain();
 
         // system-test completes successfully
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
 
         // staging-test fails multiple times, exhausts retries and failure is recorded
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, false);
+        tester.deployAndNotify(app, applicationPackage, false, DeploymentJobs.JobType.stagingTest);
         tester.buildSystem().takeJobsToRun();
         tester.clock().advance(Duration.ofMinutes(10));
         tester.notifyJobCompletion(DeploymentJobs.JobType.stagingTest, app, false);
@@ -282,17 +282,17 @@ public class UpgraderTest {
         // Application is on 5.0
         Application app = tester.createApplication("app1", "tenant1", 1, 11L);
         tester.notifyJobCompletion(DeploymentJobs.JobType.component, app, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionCorpUsEast1);
 
         // Canary in prod.corp-us-east-1 is upgraded to controller version
         tester.upgrader().maintain();
         assertEquals("Upgrade started", 1, tester.buildSystem().jobs().size());
         assertEquals(Vtag.currentVersion, ((Change.VersionChange) tester.application(app.id()).deploying().get()).version());
-        tester.deployAndNotify(DeploymentJobs.JobType.systemTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.stagingTest, app, applicationPackage, true);
-        tester.deployAndNotify(DeploymentJobs.JobType.productionCorpUsEast1, app, applicationPackage, true);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.systemTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.stagingTest);
+        tester.deployAndNotify(app, applicationPackage, true, DeploymentJobs.JobType.productionCorpUsEast1);
 
         // System is upgraded to newer version, no upgrade triggered for canary as version is lower than controller
         version = Version.fromString("5.1");