Deployment target tested on demand, but the code is a MESS!

8 files changed, 200 insertions, 102 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
index 9250bffd7d0..e3ac4cb49f2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/JobStatus.java
@@ -186,8 +186,31 @@ public class JobStatus {
         public Instant at() { return at; }
 
         @Override
-        public String toString() { return "job run " + id + " of version " + version + " "
-                                          + applicationVersion + " at " + at; }
+        public String toString() {
+            return "job run " + id + " of version " + version + " " + applicationVersion + " at " + at;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (!(o instanceof JobRun)) return false;
+
+            JobRun run = (JobRun) o;
+
+            if (id != run.id) return false;
+            if (!version.equals(run.version)) return false;
+            if (!applicationVersion.equals(run.applicationVersion)) return false;
+            return at.equals(run.at);
+        }
+
+        @Override
+        public int hashCode() {
+            int result = (int) (id ^ (id >>> 32));
+            result = 31 * result + version.hashCode();
+            result = 31 * result + applicationVersion.hashCode();
+            result = 31 * result + at.hashCode();
+            return result;
+        }
 
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 653d1bc8f22..80a325e3dc8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -1,10 +1,11 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.yahoo.collections.ArraySet;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.DeploymentSpec.Step;
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.Environment;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.ApplicationController;
@@ -29,7 +30,6 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
-import java.util.EnumSet;
 import java.util.List;
 import java.util.Map;
 import java.util.NoSuchElementException;
@@ -44,12 +44,16 @@ import java.util.stream.Stream;
 
 import static com.yahoo.config.provision.Environment.prod;
 import static com.yahoo.config.provision.Environment.staging;
+import static com.yahoo.config.provision.Environment.test;
 import static com.yahoo.vespa.hosted.controller.api.integration.BuildService.BuildJob;
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.component;
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
+import static java.util.Collections.emptySet;
+import static java.util.Collections.singletonList;
 import static java.util.Comparator.comparing;
 import static java.util.Comparator.naturalOrder;
+import static java.util.Optional.empty;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.joining;
 import static java.util.stream.Collectors.partitioningBy;
@@ -154,7 +158,7 @@ public class DeploymentTrigger {
      * the project id is removed from the application owning the job, to prevent further trigger attemps.
      */
     public boolean trigger(Job job) {
-        log.log(LogLevel.INFO, String.format("Attempting to trigger %s, deploying %s: %s (platform: %s, application: %s)", job, job.change, job.reason, job.platformVersion, job.applicationVersion.id()));
+        log.log(LogLevel.INFO, String.format("Attempting to trigger %s: %s (platform: %s, application: %s)", job, job.reason, job.platformVersion, job.applicationVersion.id()));
 
         try {
             buildService.trigger(job);
@@ -224,26 +228,23 @@ public class DeploymentTrigger {
 
     public void forceTrigger(ApplicationId applicationId, JobType jobType) {
         Application application = applications().require(applicationId);
-        if (jobType == component)
+        if (jobType == component) {
             buildService.trigger(BuildJob.of(applicationId, application.deploymentJobs().projectId().getAsLong(), jobType.jobName()));
-        else if (isVerified(application, jobType))
-            trigger(deploymentJob(application, jobType, ">:o:< Triggered by force! (-o-) |-o-| (=oo=) ", clock.instant(), Collections.emptySet()));
-        // TODO jvenstad: No need to throw here, since it will rather trigger test jobs (soon!)
+            return;
+        }
+        State target = targetFor(application, application.change(), jobType);
+        if (isVerified(application, target, jobType))
+            trigger(deploymentJob(application, target, application.change(), jobType, ">:o:< Triggered by force! (-o-) |-o-| (=oo=) ", clock.instant(), Collections.emptySet()));
+        else
+            ; // TODO jvenstad: Test it!
     }
 
-    private Job deploymentJob(Application application, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith) {
+    private Job deploymentJob(Application application, State target, Change change, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith) {
         boolean isRetry = application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::jobError)
                                      .filter(JobError.outOfCapacity::equals).isPresent();
         if (isRetry) reason += "; retrying on out of capacity";
 
-        return new Job(application, jobType, reason, availableSince, concurrentlyWith, isRetry, application.change(),
-                       targetPlatform(application, application.change(), jobType), targetApplication(application, application.change(), jobType));
-    }
-
-    private ApplicationVersion targetApplication(Application application, Change change, JobType jobType) {
-        return max(deploymentFor(application, jobType).map(Deployment::applicationVersion), change.application())
-                .orElse(application.oldestDeployedApplication()
-                                   .orElse(application.deploymentJobs().jobStatus().get(component).lastSuccess().get().applicationVersion()));
+        return new Job(application, target, jobType, reason, availableSince, concurrentlyWith, isRetry, change.application().isPresent());
     }
 
     private Version targetPlatform(Application application, Change change, JobType jobType) {
@@ -252,6 +253,12 @@ public class DeploymentTrigger {
                                    .orElse(controller.systemVersion()));
     }
 
+    private ApplicationVersion targetApplication(Application application, Change change, JobType jobType) {
+        return max(deploymentFor(application, jobType).map(Deployment::applicationVersion), change.application())
+                .orElse(application.oldestDeployedApplication()
+                                   .orElse(application.deploymentJobs().jobStatus().get(component).lastSuccess().get().applicationVersion()));
+    }
+
     private static <T extends Comparable<T>> Optional<T> max(Optional<T> o1, Optional<T> o2) {
         return ! o1.isPresent() ? o2 : ! o2.isPresent() ? o1 : o1.get().compareTo(o2.get()) >= 0 ? o1 : o2;
     }
@@ -262,58 +269,89 @@ public class DeploymentTrigger {
     private List<Job> computeReadyJobs(ApplicationId id) {
         List<Job> jobs = new ArrayList<>();
         applications().get(id).ifPresent(application -> {
-            List<DeploymentSpec.Step> steps = application.deploymentSpec().steps().isEmpty()
-                    ? Collections.singletonList(new DeploymentSpec.DeclaredZone(Environment.test))
+            List<Step> steps = application.deploymentSpec().steps().isEmpty()
+                    ? singletonList(new DeploymentSpec.DeclaredZone(test))
                     : application.deploymentSpec().steps();
-
-            Optional<Instant> completedAt = Optional.of(clock.instant());
-            String reason = "Deploying " + application.change();
-
-            for (DeploymentSpec.Step step : steps) {
+            List<Step> productionSteps = new ArrayList<>(steps);
+            productionSteps.removeIf(step -> ! step.deploysTo(prod) && ! step.zones().isEmpty());
+            List<Step> testSteps = new ArrayList<>(steps);
+            testSteps.removeAll(productionSteps);
+
+            boolean complete = ! productionSteps.isEmpty();
+            Optional<Instant> completedAt = application.deploymentJobs().statusOf(stagingTest).flatMap(JobStatus::lastSuccess).map(JobRun::at);
+            String reason = "New change available";
+            State testTarget = null;
+            Instant testAvailableSince = null;
+            JobType testFor = null;
+
+            // Loop through all production steps and find
+            //   a) any incomplete and ready production jobs which are already verified, and / or
+            //   b) any tests which are required to verify the first incomplete production job
+            for (Step step : productionSteps) {
                 Set<JobType> stepJobs = step.zones().stream().map(order::toJob).collect(toSet());
-                Set<JobType> remainingJobs = stepJobs.stream().filter(job -> ! completedAt(application.change(), application, job).isPresent()).collect(toSet());
-                if (remainingJobs.isEmpty()) { // All jobs are complete -- find the time of completion of this step.
+                Map<Optional<Instant>, List<JobType>> jobsByCompletion = stepJobs.stream().collect(groupingBy(job -> completedAt(application.change(), application, job)));
+                if (jobsByCompletion.containsKey(empty())) { // Step not complete, because some jobs remain -- trigger these if the previous step was done.
+                    for (JobType job : jobsByCompletion.get(empty())) {
+                        State target = targetFor(application, application.change(), job);
+                        if (isVerified(application, target, job)) {
+                            if (completedAt.isPresent())
+                                jobs.add(deploymentJob(application, target, application.change(), job, reason, completedAt.get(), stepJobs));
+                        }
+                        else if (testTarget == null) {
+                            testTarget = target;
+                            testAvailableSince = completedAt.orElse(clock.instant());
+                            testFor = job;
+                        }
+                    }
+
+                    complete = false;
+                    completedAt = empty();
+                    break;
+                }
+                else { // All jobs are complete -- find the time of completion of this step.
                     if (stepJobs.isEmpty()) { // No jobs means this is delay step.
                         Duration delay = ((DeploymentSpec.Delay) step).duration();
                         completedAt = completedAt.map(at -> at.plus(delay)).filter(at -> ! at.isAfter(clock.instant()));
                         reason += " after a delay of " + delay;
                     }
                     else {
-                        completedAt = stepJobs.stream().map(job -> completedAt(application.change(), application, job).get()).max(naturalOrder());
+                        completedAt = jobsByCompletion.keySet().stream().map(Optional::get).max(naturalOrder());
                         reason = "Available change in " + stepJobs.stream().map(JobType::jobName).collect(joining(", "));
                     }
                 }
-                else if (completedAt.isPresent()) { // Step not complete, because some jobs remain -- trigger these if the previous step was done.
-                    for (JobType job : remainingJobs)
-                        if (isVerified(application, job))
-                            jobs.add(deploymentJob(application, job, reason, completedAt.get(), stepJobs));
-                    completedAt = Optional.empty();
-                    break;
-                }
             }
+
+            if (productionSteps.isEmpty()) {
+                testTarget = targetFor(application, application.change(), systemTest);
+                testFor = systemTest;
+                testAvailableSince = clock.instant();
+            }
+            if (testTarget != null)
+                for (Step step : testSteps) {
+                    for (JobType jobType : step.zones().stream().map(order::toJob).collect(Collectors.toList())) {
+                        Optional<JobRun> completion = successOn(application, jobType, testTarget.targetPlatform, testTarget.targetApplication);
+                        if (completion.isPresent())
+                            testAvailableSince = completion.get().at();
+                        else if (isVerified(application, testTarget, jobType))
+                            jobs.add(deploymentJob(application, testTarget, application.change(), jobType, "Testing deployment for " + testFor.jobName(), testAvailableSince, emptySet()));
+                    }
+                }
+
             // TODO jvenstad: Replace with completion of individual parts of Change.
-            if (completedAt.isPresent())
+            if (complete)
                 applications().lockIfPresent(id, lockedApplication -> applications().store(lockedApplication.withChange(Change.empty())));
         });
         return jobs;
     }
 
-    private boolean isVerified(Application application, JobType jobType) {
-        Version targetPlatform = targetPlatform(application, application.change(), jobType);
-        ApplicationVersion targetApplication = targetApplication(application, application.change(), jobType);
+    private boolean isVerified(Application application, State state, JobType jobType) {
         if (jobType.environment() == staging)
-            return ! JobList.from(application).type(systemTest)
-                            .lastSuccess().on(targetPlatform)
-                            .lastSuccess().on(targetApplication)
-                            .isEmpty();
+            return successOn(application, systemTest, state.targetPlatform, state.targetApplication).isPresent();
         if (jobType.environment() == prod)
-            return    ! JobList.from(application).type(stagingTest)
-                               .lastSuccess().on(targetPlatform)
-                               .lastSuccess().on(targetApplication)
-                               .isEmpty()
+            return      successOn(application, stagingTest, state.targetPlatform, state.targetApplication).isPresent()
                    || ! JobList.from(application).production()
-                               .lastTriggered().on(targetPlatform)
-                               .lastTriggered().on(targetApplication)
+                               .lastTriggered().on(state.targetPlatform)
+                               .lastTriggered().on(state.targetApplication)
                                .isEmpty();
         return true;
     }
@@ -327,9 +365,7 @@ public class DeploymentTrigger {
      * part is a downgrade, regardless of the status of the job.
      */
     private Optional<Instant> completedAt(Change change, Application application, JobType jobType) {
-        Optional<JobRun> lastSuccess = application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess)
-                                                  .filter(last ->    change.platform().map(last.version()::equals).orElse(true)
-                                                                  && change.application().map(last.applicationVersion()::equals).orElse(true));
+        Optional<JobRun> lastSuccess = successOn(application, jobType, targetPlatform(application, change, jobType), targetApplication(application, change, jobType));
         if (lastSuccess.isPresent() || ! jobType.isProduction())
             return lastSuccess.map(JobRun::at);
 
@@ -341,6 +377,11 @@ public class DeploymentTrigger {
                 .map(Deployment::at);
     }
 
+    private Optional<JobRun> successOn(Application application, JobType jobType, Version targetPlatform, ApplicationVersion targetApplication) {
+        return application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess)
+                          .filter(last -> targetPlatform.equals(last.version()) && targetApplication.equals(last.applicationVersion()));
+    }
+
     private boolean canTrigger(Job job) {
         Application application = applications().require(job.applicationId());
         if (isRunning(application, job.jobType))
@@ -380,6 +421,12 @@ public class DeploymentTrigger {
         return Optional.ofNullable(application.deployments().get(jobType.zone(controller.system()).get()));
     }
 
+    private State targetFor(Application application, Change change, JobType jobType) {
+        return new State(targetPlatform(application, change, jobType),
+                         targetApplication(application, change, jobType),
+                         deploymentFor(application, jobType).map(Deployment::version),
+                         deploymentFor(application, jobType).map(Deployment::applicationVersion));
+    }
 
     private static class Job extends BuildJob {
 
@@ -389,21 +436,19 @@ public class DeploymentTrigger {
         private final Collection<JobType> concurrentlyWith;
         private final boolean isRetry;
         private final boolean isApplicationUpgrade;
-        private final Change change;
         private final Version platformVersion;
         private final ApplicationVersion applicationVersion;
 
-        private Job(Application application, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith, boolean isRetry, Change change, Version platformVersion, ApplicationVersion applicationVersion) {
+        private Job(Application application, State target, JobType jobType, String reason, Instant availableSince, Collection<JobType> concurrentlyWith, boolean isRetry, boolean isApplicationUpgrade) {
             super(application.id(), application.deploymentJobs().projectId().getAsLong(), jobType.jobName());
             this.jobType = jobType;
             this.availableSince = availableSince;
             this.concurrentlyWith = concurrentlyWith;
             this.reason = reason;
             this.isRetry = isRetry;
-            this.isApplicationUpgrade = change.application().isPresent();
-            this.change = change;
-            this.platformVersion = platformVersion;
-            this.applicationVersion = applicationVersion;
+            this.isApplicationUpgrade = isApplicationUpgrade;
+            this.platformVersion = target.targetPlatform;
+            this.applicationVersion = target.targetApplication;
         }
 
         JobType jobType() { return jobType; }
@@ -414,5 +459,21 @@ public class DeploymentTrigger {
     }
 
 
+    private static class State {
+
+        private final Version targetPlatform;
+        private final ApplicationVersion targetApplication;
+        private final Optional<Version> sourcePlatform;
+        private final Optional<ApplicationVersion> sourceApplication;
+
+        public State(Version targetPlatform, ApplicationVersion targetApplication, Optional<Version> sourcePlatform, Optional<ApplicationVersion> sourceApplication) {
+            this.targetPlatform = targetPlatform;
+            this.targetApplication = targetApplication;
+            this.sourcePlatform = sourcePlatform;
+            this.sourceApplication = sourceApplication;
+        }
+
+    }
+
 }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index a1fab2aefa2..9c82fa62da8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -180,7 +180,7 @@ public class ControllerTest {
         JobStatus jobStatus = applications.require(app1.id()).deploymentJobs().jobStatus().get(productionCorpUsEast1);
         assertNotNull("Deployment job was not removed", jobStatus);
         assertEquals(42, jobStatus.lastCompleted().get().id());
-        assertEquals("Available change in staging-test", jobStatus.lastCompleted().get().reason());
+        assertEquals("New change available", jobStatus.lastCompleted().get().reason());
 
         // prod zone removal is allowed with override
         applicationPackage = new ApplicationPackageBuilder()
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 084ffe7cbdc..59ba9f33c56 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -34,6 +34,7 @@ import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobTy
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType.systemTest;
 import static java.util.Collections.singletonList;
+import static java.util.Optional.empty;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotEquals;
@@ -42,6 +43,7 @@ import static org.junit.Assert.assertTrue;
 /**
  * @author bratseth
  * @author mpolden
+ * @author jvenstad
  */
 public class DeploymentTriggerTest {
 
@@ -361,8 +363,8 @@ public class DeploymentTriggerTest {
         tester.completeDeploymentWithError(application, applicationPackage, BuildJob.defaultBuildNumber + 1, productionUsCentral1);
 
         // deployAndNotify doesn't actually deploy if the job fails, so we need to do that manually.
-        tester.deployAndNotify(application, Optional.empty(), false, productionUsCentral1);
-        tester.deploy(productionUsCentral1, application, Optional.empty(), false);
+        tester.deployAndNotify(application, empty(), false, productionUsCentral1);
+        tester.deploy(productionUsCentral1, application, empty(), false);
 
         ApplicationVersion appVersion1 = ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber + 1);
         assertEquals(appVersion1, app.get().deployments().get(ZoneId.from("prod.us-central-1")).applicationVersion());
@@ -379,8 +381,14 @@ public class DeploymentTriggerTest {
         Version version1 = new Version("6.2");
         tester.upgradeSystem(version1);
         tester.jobCompletion(productionUsCentral1).application(application).unsuccessful().submit();
-        // TODO jvenstad: Fails here now, because job isn't triggered any more, as deploy target is not verified.
-        tester.completeUpgrade(application, version1, applicationPackage);
+        tester.deployAndNotify(application, empty(), true, systemTest);
+        tester.deployAndNotify(application, empty(), true, stagingTest);
+        tester.deployAndNotify(application, empty(), true, productionUsCentral1);
+
+        // The last job has a different target, and the tests need to run again.
+        tester.deployAndNotify(application, empty(), true, systemTest);
+        tester.deployAndNotify(application, empty(), true, stagingTest);
+        tester.deployAndNotify(application, empty(), true, productionEuWest1);
         assertEquals(appVersion1, app.get().deployments().get(ZoneId.from("prod.us-central-1")).applicationVersion());
     }
 
@@ -438,6 +446,10 @@ public class DeploymentTriggerTest {
 
         // Change is again strictly dominated, and us-central-1 is skipped, even though it is still failing.
         tester.deployAndNotify(application, applicationPackage, false, productionUsCentral1);
+
+        // Last job has a different deployment target, so tests need to run again.
+        tester.deployAndNotify(application, empty(), true, systemTest);
+        tester.deployAndNotify(application, empty(), true, stagingTest);
         tester.deployAndNotify(application, applicationPackage, true, productionEuWest1);
         assertFalse(app.get().change().isPresent());
         assertFalse(app.get().deploymentJobs().jobStatus().get(productionUsCentral1).isSuccess());
@@ -461,9 +473,9 @@ public class DeploymentTriggerTest {
         Version v1 = new Version("6.1");
         Version v2 = new Version("6.2");
         tester.upgradeSystem(v2);
-        tester.deployAndNotify(application, Optional.empty(), true, systemTest);
-        tester.deployAndNotify(application, Optional.empty(), true, stagingTest);
-        tester.deployAndNotify(application, Optional.empty(), true, productionUsCentral1);
+        tester.deployAndNotify(application, empty(), true, systemTest);
+        tester.deployAndNotify(application, empty(), true, stagingTest);
+        tester.deployAndNotify(application, empty(), true, productionUsCentral1);
         tester.deploymentTrigger().cancelChange(application.id(), true);
         tester.deploy(productionEuWest1, application, applicationPackage);
         tester.deployAndNotify(application, applicationPackage, false, productionEuWest1);
@@ -473,22 +485,21 @@ public class DeploymentTriggerTest {
         assertEquals(v1, app.get().deployments().get(productionUsEast3.zone(main).get()).version());
 
         // New application version should run system and staging tests first against 6.2, then against 6.1.
-        // TODO jvenstad: Make triggering happen at 6.2 here, since that is the first production job. Currently it tests 6.1.
         tester.jobCompletion(component).application(application).nextBuildNumber().uploadArtifact(applicationPackage).submit();
         assertEquals(v2, app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().version());
-        tester.deployAndNotify(application, Optional.empty(), true, systemTest);
+        tester.deployAndNotify(application, empty(), true, systemTest);
         assertEquals(v2, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().version());
-        tester.deployAndNotify(application, Optional.empty(), true, stagingTest);
-        tester.deployAndNotify(application, Optional.empty(), true, productionUsCentral1);
+        tester.deployAndNotify(application, empty(), true, stagingTest);
+        tester.deployAndNotify(application, empty(), true, productionUsCentral1);
         assertEquals(v1, app.get().deploymentJobs().jobStatus().get(systemTest).lastTriggered().get().version());
-        tester.deployAndNotify(application, Optional.empty(), true, systemTest);
+        tester.deployAndNotify(application, empty(), true, systemTest);
         assertEquals(v1, app.get().deploymentJobs().jobStatus().get(stagingTest).lastTriggered().get().version());
-        tester.deployAndNotify(application, Optional.empty(), true, stagingTest);
+        tester.deployAndNotify(application, empty(), true, stagingTest);
 
         // The production job on version 6.2 fails and must retry -- this is OK, even though staging now has a different version.
-        tester.deployAndNotify(application, Optional.empty(), false, productionEuWest1);
-        tester.deployAndNotify(application, Optional.empty(), true, productionUsEast3);
-        tester.deployAndNotify(application, Optional.empty(), true, productionEuWest1);
+        tester.deployAndNotify(application, empty(), false, productionEuWest1);
+        tester.deployAndNotify(application, empty(), true, productionUsEast3);
+        tester.deployAndNotify(application, empty(), true, productionEuWest1);
         assertFalse(app.get().change().isPresent());
         assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionUsCentral1).lastSuccess().get().applicationVersion().buildNumber().get().longValue());
         assertEquals(43, app.get().deploymentJobs().jobStatus().get(productionEuWest1).lastSuccess().get().applicationVersion().buildNumber().get().longValue());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index db95a15cff4..5b1c99ec727 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -504,6 +504,9 @@ public class UpgraderTest {
         tester.readyJobTrigger().maintain();
         tester.readyJobTrigger().maintain();
         tester.readyJobTrigger().maintain();
+        tester.readyJobTrigger().maintain();
+        tester.readyJobTrigger().maintain();
+        tester.readyJobTrigger().maintain();
 
         // Canaries upgrade and raise confidence of V2
         tester.completeUpgrade(canary0, v2, "canary");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
index e1ff2712e4a..1a88a7691f1 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
@@ -48,7 +48,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.101-commit1",
+        "reason": "Testing deployment for production-us-east-3",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -62,7 +62,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.101-commit1",
+        "reason": "Testing deployment for production-us-east-3",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -76,7 +76,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.101-commit1",
+        "reason": "Testing deployment for production-us-east-3",
         "at": "(ignore)"
       }
     },
@@ -94,7 +94,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-us-west-1",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -108,7 +108,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-us-west-1",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -122,7 +122,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-us-west-1",
         "at": "(ignore)"
       }
     },
@@ -140,7 +140,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -154,7 +154,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -168,7 +168,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       }
     },
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
index 2491aec22d1..e3d7c86c051 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
@@ -58,7 +58,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.42-commit1",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -72,7 +72,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.42-commit1",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -86,7 +86,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.42-commit1",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       }
     },
@@ -104,7 +104,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -118,7 +118,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -132,7 +132,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       }
     },
@@ -150,7 +150,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -164,7 +164,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       },
       "firstFailing": {
@@ -178,7 +178,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       }
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
index 8598ff37333..4bc7511c695 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
@@ -58,7 +58,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.42-commit1",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -72,7 +72,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.42-commit1",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -86,7 +86,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Deploying application change to 1.0.42-commit1",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       }
     },
@@ -104,7 +104,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -118,7 +118,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       },
       "lastSuccess": {
@@ -132,7 +132,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in system-test",
+        "reason": "Testing deployment for production-corp-us-east-1",
         "at": "(ignore)"
       }
     },
@@ -150,7 +150,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       },
       "lastCompleted": {
@@ -164,7 +164,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       },
       "firstFailing": {
@@ -178,7 +178,7 @@
             "gitCommit": "commit1"
           }
         },
-        "reason": "Available change in staging-test",
+        "reason": "New change available",
         "at": "(ignore)"
       }
     }