Merge pull request #20892 from vespa-engine/jonmv/deployment-orchestration-for-long-pipelines

Jonmv/deployment orchestration for long pipelines

8 files changed, 224 insertions, 65 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
index 6e31c93dbdd..3fe5240ce34 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java
@@ -45,15 +45,17 @@ public class Instance {
     private final RotationStatus rotationStatus;
     private final Map<JobType, Instant> jobPauses;
     private final Change change;
+    private final Optional<ApplicationVersion> latestDeployed;
 
     /** Creates an empty instance */
     public Instance(ApplicationId id) {
-        this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty());
+        this(id, Set.of(), Map.of(), List.of(), RotationStatus.EMPTY, Change.empty(), Optional.empty());
     }
 
     /** Creates an empty instance*/
     public Instance(ApplicationId id, Collection<Deployment> deployments, Map<JobType, Instant> jobPauses,
-                    List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change) {
+                    List<AssignedRotation> rotations, RotationStatus rotationStatus, Change change,
+                    Optional<ApplicationVersion> latestDeployed) {
         this.id = Objects.requireNonNull(id, "id cannot be null");
         this.deployments = Objects.requireNonNull(deployments, "deployments cannot be null").stream()
                                   .collect(Collectors.toUnmodifiableMap(Deployment::zone, Function.identity()));
@@ -61,6 +63,7 @@ public class Instance {
         this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null"));
         this.rotationStatus = Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null");
         this.change = Objects.requireNonNull(change, "change cannot be null");
+        this.latestDeployed = Objects.requireNonNull(latestDeployed, "latestDeployed cannot be null");
     }
 
     public Instance withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version,
@@ -87,7 +90,7 @@ public class Instance {
         else
             jobPauses.remove(jobType);
 
-        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change);
+        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed);
     }
 
     public Instance recordActivityAt(Instant instant, ZoneId zone) {
@@ -118,15 +121,19 @@ public class Instance {
     }
 
     public Instance with(List<AssignedRotation> assignedRotations) {
-        return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change);
+        return new Instance(id, deployments.values(), jobPauses, assignedRotations, rotationStatus, change, latestDeployed);
     }
 
     public Instance with(RotationStatus rotationStatus) {
-        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change);
+        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed);
     }
 
     public Instance withChange(Change change) {
-        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change);
+        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed);
+    }
+
+    public Instance withLatestDeployed(ApplicationVersion latestDeployed) {
+        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, Optional.of(latestDeployed));
     }
 
     private Instance with(Deployment deployment) {
@@ -136,7 +143,7 @@ public class Instance {
     }
 
     private Instance with(Map<ZoneId, Deployment> deployments) {
-        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change);
+        return new Instance(id, deployments.values(), jobPauses, rotations, rotationStatus, change, latestDeployed);
     }
 
     public ApplicationId id() { return id; }
@@ -181,6 +188,11 @@ public class Instance {
         return change;
     }
 
+    /** Returns the application version that last completedd roll-out to this instance. */
+    public Optional<ApplicationVersion> latestDeployed() {
+        return latestDeployed;
+    }
+
     /** Returns the total quota usage for this instance, excluding temporary deployments **/
     public QuotaUsage quotaUsage() {
         return deployments.values().stream()
@@ -199,7 +211,7 @@ public class Instance {
     @Override
     public boolean equals(Object o) {
         if (this == o) return true;
-        if (! (o instanceof Instance)) return false;
+        if ( ! (o instanceof Instance)) return false;
 
         Instance that = (Instance) o;
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index e7521b37dbf..ad237a6aa6e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -13,6 +13,7 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.Change;
@@ -26,7 +27,6 @@ import java.util.Collections;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
-import java.util.Objects;
 import java.util.Optional;
 import java.util.function.Function;
 import java.util.stream.Collectors;
@@ -216,12 +216,29 @@ public class DeploymentStatus {
      * and has not yet started rolling out, due to some other change or a block window being present at the time of submission.
      */
     public Change outstandingChange(InstanceName instance) {
-        return application.latestVersion().map(Change::of)
+        return nextVersion(instance).map(Change::of)
                           .filter(change -> application.require(instance).change().application().map(change::upgrades).orElse(true))
                           .filter(change -> ! jobsToRun(Map.of(instance, change)).isEmpty())
                           .orElse(Change.empty());
     }
 
+    /** The next application version to roll out to instance. */
+    private Optional<ApplicationVersion> nextVersion(InstanceName instance) {
+        return Optional.ofNullable(instanceSteps().get(instance)).stream()
+                       .flatMap(this::allDependencies)
+                       .flatMap(step -> step.instance.latestDeployed().stream())
+                       .min(naturalOrder())
+                       .or(application::latestVersion);
+    }
+
+    private Stream<InstanceStatus> allDependencies(StepStatus step) {
+        return step.dependencies.stream()
+                                .flatMap(dep -> Stream.concat(Stream.of(dep), allDependencies(dep)))
+                                .filter(InstanceStatus.class::isInstance)
+                                .map(InstanceStatus.class::cast)
+                                .distinct();
+    }
+
     /**
      * True if the job has already been triggered on the given versions, or if all test types (systemTest, stagingTest),
      * restricted to the job's instance if declared in that instance, have successful runs on the given versions.
@@ -237,7 +254,7 @@ public class DeploymentStatus {
     }
 
     private Map<JobId, Versions> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) {
-        ImmutableMap.Builder<JobId, Versions> jobs = ImmutableMap.builder();
+        Map<JobId, Versions> jobs = new LinkedHashMap<>();
         jobSteps.forEach((job, step) -> {
             // When computing eager test jobs for outstanding changes, assume current upgrade completes successfully.
             Optional<Deployment> deployment = deploymentFor(job)
@@ -252,15 +269,10 @@ public class DeploymentStatus {
                                                            : existing);
             if (   job.application().instance().equals(instance)
                 && job.type().isProduction()
-                && step.completedAt(change).isEmpty())
+                && step.completedAt(change, Optional.of(job)).isEmpty()) // Signal strict completion criterion by depending on job itself.
                 jobs.put(job, Versions.from(change, application, deployment, systemVersion));
         });
-        return jobs.build();
-    }
-
-    /** The production jobs that need to run to complete roll-out of the given change to production. */
-    public Map<JobId, Versions> productionJobs(InstanceName instance, Change change) {
-        return productionJobs(instance, change, false);
+        return jobs;
     }
 
     /** The test jobs that need to run prior to the given production deployment jobs. */
@@ -315,7 +327,7 @@ public class DeploymentStatus {
     /** Adds the primitive steps contained in the given step, which depend on the given previous primitives, to the dependency graph. */
     private List<StepStatus> fillStep(Map<JobId, StepStatus> dependencies, List<StepStatus> allSteps,
                                       DeploymentSpec.Step step, List<StepStatus> previous, InstanceName instance) {
-        if (step.steps().isEmpty()) {
+        if (step.steps().isEmpty() && ! (step instanceof DeploymentInstanceSpec)) {
             if (instance == null)
                 return previous; // Ignore test and staging outside all instances.
 
@@ -417,7 +429,7 @@ public class DeploymentStatus {
 
         private final StepType type;
         private final DeploymentSpec.Step step;
-        private final List<StepStatus> dependencies;
+        private final List<StepStatus> dependencies; // All direct dependencies of this step.
         private final InstanceName instance;
 
         private StepStatus(StepType type, DeploymentSpec.Step step, List<StepStatus> dependencies, InstanceName instance) {
@@ -492,7 +504,7 @@ public class DeploymentStatus {
         }
 
         @Override
-        public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
+        Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
             return readyAt(change, dependent).map(completion -> completion.plus(step().delay()));
         }
 
@@ -520,13 +532,14 @@ public class DeploymentStatus {
          * for this instance, or if no more jobs should run for this instance for the given change.
          */
         @Override
-        public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
+        Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
             return    (   (change.platform().isEmpty() || change.platform().equals(instance.change().platform()))
                        && (change.application().isEmpty() || change.application().equals(instance.change().application()))
                    || status.jobsToRun(Map.of(instance.name(), change)).isEmpty())
                       ? dependenciesCompletedAt(change, dependent)
                       : Optional.empty();
         }
+        // TODO jonmv: complete for p-jobs: last is XXX, but ready/verified uses any is XXX.
 
         @Override
         public Optional<Instant> blockedUntil(Change change) {
@@ -607,13 +620,15 @@ public class DeploymentStatus {
 
                 /** Complete if deployment is on pinned version, and last successful deployment, or if given versions is strictly a downgrade, and this isn't forced by a pin. */
                 @Override
-                public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
+                Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
                     if (     change.isPinned()
                         &&   change.platform().isPresent()
                         && ! existingDeployment.map(Deployment::version).equals(change.platform()))
                         return Optional.empty();
 
-                    if (change.application().isPresent() && ! existingDeployment.map(Deployment::applicationVersion).equals(change.application()))
+                    if (     change.application().isPresent()
+                        && ! existingDeployment.map(Deployment::applicationVersion).equals(change.application())
+                        &&   dependent.equals(job())) // Job should (re-)run in this case, but other dependents need not wait.
                         return Optional.empty();
 
                     Change fullChange = status.application().require(instance).change();
@@ -622,10 +637,12 @@ public class DeploymentStatus {
                                           .orElse(false))
                         return job.lastCompleted().flatMap(Run::end);
 
-                    return job.lastSuccess()
-                              .filter(run ->    change.platform().map(run.versions().targetPlatform()::equals).orElse(true)
-                                             && change.application().map(run.versions().targetApplication()::equals).orElse(true))
-                              .flatMap(Run::end);
+                    return (dependent.equals(job()) ? job.lastSuccess().stream()
+                                                    : RunList.from(job).status(RunStatus.success).asList().stream())
+                            .filter(run ->    change.platform().map(run.versions().targetPlatform()::equals).orElse(true)
+                                           && change.application().map(run.versions().targetApplication()::equals).orElse(true))
+                            .findFirst()
+                            .flatMap(Run::end);
                 }
             };
         }
@@ -635,16 +652,21 @@ public class DeploymentStatus {
             JobStatus job = status.instanceJobs(instance).get(testType);
             return new JobStepStatus(StepType.test, step, dependencies, job, status) {
                 @Override
-                public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
+                Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
                     Versions versions = Versions.from(change, status.application, status.deploymentFor(job.id()), status.systemVersion);
-                    return job.lastSuccess()
-                              .filter(run -> versions.targetsMatch(run.versions()))
-                              .filter(run -> ! status.jobs()
-                                                     .instance(instance)
-                                                     .type(prodType)
-                                                     .lastCompleted().endedNoLaterThan(run.start())
-                                                     .isEmpty())
-                              .map(run -> run.end().get());
+                    return dependent.equals(job()) ? job.lastSuccess()
+                                                        .filter(run -> versions.targetsMatch(run.versions()))
+                                                        .filter(run -> ! status.jobs()
+                                                                               .instance(instance)
+                                                                               .type(prodType)
+                                                                               .lastCompleted().endedNoLaterThan(run.start())
+                                                                               .isEmpty())
+                                                        .map(run -> run.end().get())
+                                                   : RunList.from(job)
+                                                            .matching(run -> versions.targetsMatch(run.versions()))
+                                                            .status(RunStatus.success)
+                                                            .first()
+                                                            .map(run -> run.end().get());
                 }
             };
         }
@@ -655,7 +677,7 @@ public class DeploymentStatus {
             JobStatus job = status.instanceJobs(instance).get(jobType);
             return new JobStepStatus(StepType.test, step, dependencies, job, status) {
                 @Override
-                public Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
+                Optional<Instant> completedAt(Change change, Optional<JobId> dependent) {
                     return RunList.from(job)
                                   .matching(run -> run.versions().targetsMatch(Versions.from(change,
                                                                                              status.application,
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
index c6e4d02acfa..22df5ca559e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatusList.java
@@ -24,7 +24,7 @@ public class DeploymentStatusList extends AbstractFilteringList<DeploymentStatus
 
     /** Returns the subset of applications which have changes left to deploy; blocked, or deploying */
     public DeploymentStatusList withChanges() {
-        return matching(status -> status.application().instances().values().stream()
+        return matching(status -> status.application().productionInstances().values().stream()
                                         .anyMatch(instance -> instance.change().hasTargets() || status.outstandingChange(instance.name()).hasTargets()));
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 4e8f17b6098..d3f2d4e4501 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -99,10 +99,7 @@ public class DeploymentTrigger {
                              .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false)
                     && acceptNewApplicationVersion(status, instanceName)) {
                     application = application.with(instanceName,
-                                                   instance -> {
-                                                       instance = instance.withChange(instance.change().with(outstanding.application().get()));
-                                                       return instance.withChange(remainingChange(instance, status));
-                                                   });
+                                                   instance -> withRemainingChange(instance, instance.change().with(outstanding.application().get()), status));
                 }
             }
             applications().store(application);
@@ -121,7 +118,7 @@ public class DeploymentTrigger {
 
         applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application ->
                 applications().store(application.with(id.instance(),
-                                                      instance -> instance.withChange(remainingChange(instance, jobs.deploymentStatus(application.get()))))));
+                                                      instance -> withRemainingChange(instance, instance.change(), jobs.deploymentStatus(application.get())))));
     }
 
     /**
@@ -277,13 +274,8 @@ public class DeploymentTrigger {
     /** Overrides the given instance's platform and application changes with any contained in the given change. */
     public void forceChange(ApplicationId instanceId, Change change) {
         applications().lockApplicationOrThrow(TenantAndApplicationId.from(instanceId), application -> {
-            Change newChange = change.onTopOf(application.get().require(instanceId.instance()).change());
-            application = application.with(instanceId.instance(),
-                                           instance -> instance.withChange(newChange));
-            DeploymentStatus newStatus = jobs.deploymentStatus(application.get());
-            application = application.with(instanceId.instance(),
-                                           instance -> instance.withChange(remainingChange(instance, newStatus)));
-            applications().store(application);
+            applications().store(application.with(instanceId.instance(),
+                                                  instance -> withRemainingChange(instance, change.onTopOf(application.get().require(instanceId.instance()).change()), jobs.deploymentStatus(application.get()))));
         });
     }
 
@@ -300,7 +292,7 @@ public class DeploymentTrigger {
                 default: throw new IllegalArgumentException("Unknown cancellation choice '" + cancellation + "'!");
             }
             applications().store(application.with(instanceId.instance(),
-                                                  instance -> instance.withChange(change)));
+                                                  instance -> withRemainingChange(instance, change, jobs.deploymentStatus(application.get()))));
         });
     }
 
@@ -379,13 +371,16 @@ public class DeploymentTrigger {
         return status.application().require(instance).change().platform().isEmpty();
     }
 
-    private Change remainingChange(Instance instance, DeploymentStatus status) {
-        Change change = instance.change();
-        if (status.jobsToRun(Map.of(instance.name(), instance.change().withoutApplication())).isEmpty())
-            change = change.withoutPlatform();
-        if (status.jobsToRun(Map.of(instance.name(), instance.change().withoutPlatform())).isEmpty())
-            change = change.withoutApplication();
-        return change;
+    private Instance withRemainingChange(Instance instance, Change change, DeploymentStatus status) {
+        Change remaining = change;
+        if (status.jobsToRun(Map.of(instance.name(), change.withoutApplication())).isEmpty())
+            remaining = remaining.withoutPlatform();
+        if (status.jobsToRun(Map.of(instance.name(), change.withoutPlatform())).isEmpty()) {
+            remaining = remaining.withoutApplication();
+            if (change.application().isPresent())
+                instance = instance.withLatestDeployed(change.application().get());
+        }
+        return instance.withChange(remaining);
     }
 
     // ---------- Version and job helpers ----------
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
index cec2e698e9e..5021d6b2076 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobList.java
@@ -85,12 +85,12 @@ public class JobList extends AbstractFilteringList<JobStatus, JobList> {
         return matching(job -> job.id().type().isProduction());
     }
 
-    /** Returns the jobs with any runs matching the given versions — targets only for system test, everything present otherwise. */
+    /** Returns the jobs with any runs matching the given versions — targets only for system test, everything present otherwise. */
     public JobList triggeredOn(Versions versions) {
         return matching(job -> ! RunList.from(job).on(versions).isEmpty());
     }
 
-    /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */
+    /** Returns the jobs with successful runs matching the given versions — targets only for system test, everything present otherwise. */
     public JobList successOn(Versions versions) {
         return matching(job -> ! RunList.from(job).status(RunStatus.success).on(versions).isEmpty());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index 4b060846090..a7413273a38 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -94,6 +94,7 @@ public class ApplicationSerializer {
     private static final String deploymentJobsField = "deploymentJobs"; // TODO jonmv: clean up serialisation format
     private static final String assignedRotationsField = "assignedRotations";
     private static final String assignedRotationEndpointField = "endpointId";
+    private static final String latestDeployedField = "latestDeployed";
 
     // Deployment fields
     private static final String zoneField = "zone";
@@ -176,6 +177,7 @@ public class ApplicationSerializer {
             assignedRotationsToSlime(instance.rotations(), instanceObject);
             toSlime(instance.rotationStatus(), instanceObject.setArray(rotationStatusField));
             toSlime(instance.change(), instanceObject, deployingField);
+            instance.latestDeployed().ifPresent(version -> toSlime(version, instanceObject.setObject(latestDeployedField)));
         }
     }
 
@@ -330,12 +332,14 @@ public class ApplicationSerializer {
             List<AssignedRotation> assignedRotations = assignedRotationsFromSlime(object);
             RotationStatus rotationStatus = rotationStatusFromSlime(object);
             Change change = changeFromSlime(object.field(deployingField));
+            Optional<ApplicationVersion> latestDeployed = latestVersionFromSlime(object.field(latestDeployedField));
             instances.add(new Instance(id.instance(instanceName),
                                        deployments,
                                        jobPauses,
                                        assignedRotations,
                                        rotationStatus,
-                                       change));
+                                       change,
+                                       latestDeployed));
         });
         return instances;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index c7c434910f3..52521775ddd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -468,7 +468,11 @@ public class DeploymentTriggerTest {
     public void downgradingApplicationVersionWorks() {
         var app = tester.newDeploymentContext().submit().deploy();
         ApplicationVersion appVersion0 = app.lastSubmission().get();
+        assertEquals(Optional.of(appVersion0), app.instance().latestDeployed());
+
         app.submit().deploy();
+        ApplicationVersion appVersion1 = app.lastSubmission().get();
+        assertEquals(Optional.of(appVersion1), app.instance().latestDeployed());
 
         // Downgrading application version.
         tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion0));
@@ -479,19 +483,27 @@ public class DeploymentTriggerTest {
            .runJob(productionUsWest1);
         assertEquals(Change.empty(), app.instance().change());
         assertEquals(appVersion0, app.instance().deployments().get(productionUsEast3.zone(tester.controller().system())).applicationVersion());
+        assertEquals(Optional.of(appVersion0), app.instance().latestDeployed());
     }
 
     @Test
     public void settingANoOpChangeIsANoOp() {
-        var app = tester.newDeploymentContext().submit().deploy();
+        var app = tester.newDeploymentContext().submit();
+        assertEquals(Optional.empty(), app.instance().latestDeployed());
+
+        app.deploy();
         ApplicationVersion appVersion0 = app.lastSubmission().get();
+        assertEquals(Optional.of(appVersion0), app.instance().latestDeployed());
+
         app.submit().deploy();
         ApplicationVersion appVersion1 = app.lastSubmission().get();
+        assertEquals(Optional.of(appVersion1), app.instance().latestDeployed());
 
         // Triggering a roll-out of an already deployed application is a no-op.
         assertEquals(Change.empty(), app.instance().change());
         tester.deploymentTrigger().forceChange(app.instanceId(), Change.of(appVersion1));
         assertEquals(Change.empty(), app.instance().change());
+        assertEquals(Optional.of(appVersion1), app.instance().latestDeployed());
     }
 
     @Test
@@ -807,6 +819,115 @@ public class DeploymentTriggerTest {
     }
 
     @Test
+    public void testMultipleInstancesWithDifferentChanges() {
+        DeploymentContext i1 = tester.newDeploymentContext("t", "a", "i1");
+        DeploymentContext i2 = tester.newDeploymentContext("t", "a", "i2");
+        DeploymentContext i3 = tester.newDeploymentContext("t", "a", "i3");
+        DeploymentContext i4 = tester.newDeploymentContext("t", "a", "i4");
+        ApplicationPackage applicationPackage = ApplicationPackageBuilder
+                .fromDeploymentXml("<deployment version='1'>\n" +
+                                   "  <parallel>\n" +
+                                   "    <instance id='i1'>\n" +
+                                   "      <prod>\n" +
+                                   "        <region>us-east-3</region>\n" +
+                                   "        <delay hours='6' />\n" +
+                                   "      </prod>\n" +
+                                   "    </instance>\n" +
+                                   "    <instance id='i2'>\n" +
+                                   "      <prod>\n" +
+                                   "        <region>us-east-3</region>\n" +
+                                   "      </prod>\n" +
+                                   "    </instance>\n" +
+                                   "  </parallel>\n" +
+                                   "  <instance id='i3'>\n" +
+                                   "    <prod>\n" +
+                                   "      <region>us-east-3</region>\n" +
+                                   "        <delay hours='18' />\n" +
+                                   "      <test>us-east-3</test>\n" +
+                                   "    </prod>\n" +
+                                   "  </instance>\n" +
+                                   "  <instance id='i4'>\n" +
+                                   "    <test />\n" +
+                                   "    <staging />\n" +
+                                   "    <prod>\n" +
+                                   "      <region>us-east-3</region>\n" +
+                                   "    </prod>\n" +
+                                   "  </instance>\n" +
+                                   "</deployment>\n");
+
+        // Package is submitted, and change propagated to the two first instances.
+        i1.submit(applicationPackage);
+        Optional<ApplicationVersion> v0 = i1.lastSubmission();
+        tester.outstandingChangeDeployer().run();
+        assertEquals(v0, i1.instance().change().application());
+        assertEquals(v0, i2.instance().change().application());
+        assertEquals(Optional.empty(), i3.instance().change().application());
+        assertEquals(Optional.empty(), i4.instance().change().application());
+
+        // Tests run in i4, as they're declared there, and i1 and i2 get to work
+        i4.runJob(systemTest).runJob(stagingTest);
+        i1.runJob(productionUsEast3);
+        i2.runJob(productionUsEast3);
+
+        // Since the post-deployment delay of i1 is incomplete, i3 doesn't yet get the change.
+        tester.outstandingChangeDeployer().run();
+        assertEquals(v0, i1.instance().latestDeployed());
+        assertEquals(v0, i2.instance().latestDeployed());
+        assertEquals(Optional.empty(), i1.instance().change().application());
+        assertEquals(Optional.empty(), i2.instance().change().application());
+        assertEquals(Optional.empty(), i3.instance().change().application());
+        assertEquals(Optional.empty(), i4.instance().change().application());
+
+        // When the delay is done, i3 gets the change.
+        tester.clock().advance(Duration.ofHours(6));
+        tester.outstandingChangeDeployer().run();
+        assertEquals(Optional.empty(), i1.instance().change().application());
+        assertEquals(Optional.empty(), i2.instance().change().application());
+        assertEquals(v0, i3.instance().change().application());
+        assertEquals(Optional.empty(), i4.instance().change().application());
+
+        // v0 begins roll-out in i3, and v1 is submitted and rolls out in i1 and i2 some time later
+        i3.runJob(productionUsEast3); // v0
+        tester.clock().advance(Duration.ofHours(12));
+        i1.submit(applicationPackage);
+        Optional<ApplicationVersion> v1 = i1.lastSubmission();
+        i4.runJob(systemTest).runJob(stagingTest);
+        i1.runJob(productionUsEast3); // v1
+        i2.runJob(productionUsEast3); // v1
+        assertEquals(v1, i1.instance().latestDeployed());
+        assertEquals(v1, i2.instance().latestDeployed());
+        assertEquals(Optional.empty(), i1.instance().change().application());
+        assertEquals(Optional.empty(), i2.instance().change().application());
+        assertEquals(v0, i3.instance().change().application());
+        assertEquals(Optional.empty(), i4.instance().change().application());
+
+        // After some time, v2 also starts rolling out to i1 and i2, but does not complete in i2
+        tester.clock().advance(Duration.ofHours(3));
+        i1.submit(applicationPackage);
+        Optional<ApplicationVersion> v2 = i1.lastSubmission();
+        i4.runJob(systemTest).runJob(stagingTest);
+        i1.runJob(productionUsEast3); // v2
+        tester.clock().advance(Duration.ofHours(3));
+
+        // v1 is all done in i1 and i2, but does not yet roll out in i3; v2 is not completely rolled out there yet.
+        // TODO jonmv: thie belowh new revision policy, but must be faked for now, as v1 would not wait for v0 to complete.
+        //tester.outstandingChangeDeployer().run();
+        assertEquals(v0, i3.instance().change().application());
+
+        // i3 completes v0, which rolls out to i4; v1 is ready for i3, but v2 is not.
+        i3.runJob(testUsEast3);
+        assertEquals(Optional.empty(), i3.instance().change().application());
+        tester.outstandingChangeDeployer().run();
+        assertEquals(v2, i1.instance().latestDeployed());
+        assertEquals(v1, i2.instance().latestDeployed());
+        assertEquals(v0, i3.instance().latestDeployed());
+        assertEquals(Optional.empty(), i1.instance().change().application());
+        assertEquals(v2, i2.instance().change().application());
+        assertEquals(v1, i3.instance().change().application());
+        assertEquals(v0, i4.instance().change().application());
+    }
+
+    @Test
     public void testMultipleInstances() {
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .instances("instance1,instance2")
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index b33f8f6f7e7..86e5acf07a0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -120,13 +120,15 @@ public class ApplicationSerializerTest {
                                                         Map.of(JobType.systemTest, Instant.ofEpochMilli(333)),
                                                         List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of("us-west-1"))),
                                                         rotationStatus,
-                                                        Change.of(new Version("6.1"))),
+                                                        Change.of(new Version("6.1")),
+                                                        Optional.of(applicationVersion2)),
                                            new Instance(id3,
                                                         List.of(),
                                                         Map.of(),
                                                         List.of(),
                                                         RotationStatus.EMPTY,
-                                                        Change.of(Version.fromString("6.7")).withPin()));
+                                                        Change.of(Version.fromString("6.7")).withPin(),
+                                                        Optional.empty()));
 
         Application original = new Application(TenantAndApplicationId.from(id1),
                                                Instant.now().truncatedTo(ChronoUnit.MILLIS),
@@ -182,6 +184,9 @@ public class ApplicationSerializerTest {
         assertEquals(original.require(id1.instance()).change(), serialized.require(id1.instance()).change());
         assertEquals(original.require(id3.instance()).change(), serialized.require(id3.instance()).change());
 
+        assertEquals(original.require(id1.instance()).latestDeployed(), serialized.require(id1.instance()).latestDeployed());
+        assertEquals(original.require(id3.instance()).latestDeployed(), serialized.require(id3.instance()).latestDeployed());
+
         // Test metrics
         assertEquals(original.metrics().queryServiceQuality(), serialized.metrics().queryServiceQuality(), Double.MIN_VALUE);
         assertEquals(original.metrics().writeServiceQuality(), serialized.metrics().writeServiceQuality(), Double.MIN_VALUE);