Merge pull request #4176 from vespa-engine/jvenstad/deployment-cleanup-part-deux

Jvenstad/deployment cleanup part deux

11 files changed, 88 insertions, 137 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
index 3fafd81d10e..449558e4054 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Change.VersionChange;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 
@@ -99,10 +100,9 @@ public class Application {
      * (deployments also includes manually deployed environments)
      */
     public Map<Zone, Deployment> productionDeployments() {
-        return deployments.values().stream()
-                                   .filter(deployment -> deployment.zone().environment() == Environment.prod)
-                                   .collect(Collectors.collectingAndThen(Collectors.toMap(Deployment::zone, Function.identity()),
-                                                                         ImmutableMap::copyOf));
+        return ImmutableMap.copyOf(deployments.values().stream()
+                                           .filter(deployment -> deployment.zone().environment() == Environment.prod)
+                                           .collect(Collectors.toMap(Deployment::zone, Function.identity())));
     }
 
     public DeploymentJobs deploymentJobs() { return deploymentJobs; }
@@ -123,35 +123,24 @@ public class Application {
      * Returns the oldest version this has deployed in a permanent zone (not test or staging),
      * or empty version if it is not deployed anywhere
      */
-    public Optional<Version> deployedVersion() {
+    public Optional<Version> oldestDeployedVersion() {
         return productionDeployments().values().stream()
-                                               .sorted(Comparator.comparing(Deployment::version))
-                                               .findFirst()
-                                               .map(Deployment::version);
+                .map(Deployment::version)
+                .min(Comparator.naturalOrder());
     }
 
-    /** The version that should be used to compile this application */
-    public Version compileVersion(Controller controller) {
-        return deployedVersion().orElse(controller.systemVersion());
-    }
-
-    /** Returns the version a deployment to this zone should use for this application */
-    public Version currentDeployVersion(Controller controller, Zone zone) {
-        if ( ! deploying().isPresent())
-            return currentVersion(controller, zone);
-        else if ( deploying().get() instanceof Change.ApplicationChange)
-            return currentVersion(controller, zone);
-        else
+    /** Returns the version a new deployment to this zone should use for this application */
+    public Version deployVersionIn(Zone zone, Controller controller) {
+        if (deploying().isPresent() && deploying().get() instanceof VersionChange)
             return ((Change.VersionChange) deploying().get()).version();
+
+        return versionIn(zone, controller);
     }
 
     /** Returns the current version this application has, or if none; should use, in the given zone */
-    public Version currentVersion(Controller controller, Zone zone) {
-        Deployment currentDeployment = deployments().get(zone);
-        if (currentDeployment != null) // Already deployed in this zone: Use that version
-            return currentDeployment.version();
-
-        return deployedVersion().orElse(controller.systemVersion());
+    public Version versionIn(Zone zone, Controller controller) {
+        return Optional.ofNullable(deployments().get(zone)).map(Deployment::version) // Already deployed in this zone: Use that version
+                .orElse(oldestDeployedVersion().orElse(controller.systemVersion()));
     }
 
     @Override
@@ -174,14 +163,8 @@ public class Application {
         return "application '" + id + "'";
     }
 
-    /** Returns true if there is a current change which is blocked from being deployed to production at this instant */
-    public boolean deployingBlocked(Instant instant) {
-        if ( ! deploying.isPresent()) return false;
-        return deploying.get().blockedBy(deploymentSpec, instant);
-    }
-    
     public boolean isBlocked(Instant instant) {
-        return ! deploymentSpec.canUpgradeAt(instant) || ! deploymentSpec.canChangeRevisionAt(instant);
+         return ! deploymentSpec().canUpgradeAt(instant) || ! deploymentSpec().canChangeRevisionAt(instant);
     }
 
     public Optional<IssueId> ownershipIssueId() {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index f5bf77cd6d3..5c6cea43e76 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -288,22 +288,22 @@ public class ApplicationController {
     public ActivateResult deployApplication(ApplicationId applicationId, Zone zone,
                                             ApplicationPackage applicationPackage, DeployOptions options) {
         try (Lock lock = lock(applicationId)) {
-            // Determine what we are doing
             LockedApplication application = get(applicationId, lock).orElse(new LockedApplication(
                     new Application(applicationId), lock)
             );
 
+            // Determine what we are doing
             Version version;
             if (options.deployCurrentVersion)
-                version = application.currentVersion(controller, zone);
+                version = application.versionIn(zone, controller);
             else if (canDeployDirectlyTo(zone, options))
                 version = options.vespaVersion.map(Version::new).orElse(controller.systemVersion());
             else if ( ! application.deploying().isPresent() && ! zone.environment().isManuallyDeployed())
                 return unexpectedDeployment(applicationId, zone, applicationPackage);
             else
-                version = application.currentDeployVersion(controller, zone);
+                version = application.deployVersionIn(zone, controller);
 
-            Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.zoneRegistry().system(), zone);
+            Optional<DeploymentJobs.JobType> jobType = DeploymentJobs.JobType.from(controller.system(), zone);
             ApplicationRevision revision = toApplicationPackageRevision(applicationPackage, options.screwdriverBuildJob);
 
             if ( ! options.deployCurrentVersion) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
index f34583d3998..3ef904ebf85 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java
@@ -8,8 +8,10 @@ import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Change.ApplicationChange;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
 
 import java.time.Instant;
 import java.util.LinkedHashMap;
@@ -66,8 +68,8 @@ public class LockedApplication extends Application {
         return new LockedApplication(new Application(id(), deploymentSpec(), validationOverrides(), deployments(),
                                                      deploymentJobs().withTriggering(type,
                                                                                      change,
-                                                                                     determineTriggerVersion(type, controller),
-                                                                                     determineTriggerRevision(type, controller),
+                                                                                     deployVersionFor(type, controller),
+                                                                                     deployRevisionFor(type, controller),
                                                                                      reason,
                                                                                      triggerTime),
                                                      deploying(), hasOutstandingChange(), ownershipIssueId()), lock);
@@ -132,42 +134,29 @@ public class LockedApplication extends Application {
                                                      hasOutstandingChange(), Optional.of(issueId)), lock);
     }
 
-    private Version determineTriggerVersion(DeploymentJobs.JobType jobType, Controller controller) {
-        Optional<Zone> zone = jobType.zone(controller.system());
-        if ( ! zone.isPresent()) // a sloppy test TODO: Fix
-            return controller.systemVersion();
-        return currentDeployVersion(controller, zone.get());
+    private Version deployVersionFor(DeploymentJobs.JobType jobType, Controller controller) {
+        return jobType == JobType.component
+               ? controller.systemVersion()
+               : deployVersionIn(jobType.zone(controller.system()).get(), controller);
     }
 
-    private Optional<ApplicationRevision> determineTriggerRevision(DeploymentJobs.JobType jobType,
-                                                                   Controller controller) {
-        Optional<Zone> zone = jobType.zone(controller.system());
-        if ( ! zone.isPresent()) // a sloppy test TODO: Fix
-            return Optional.empty();
-        return currentDeployRevision(jobType.zone(controller.system()).get());
+    private Optional<ApplicationRevision> deployRevisionFor(DeploymentJobs.JobType jobType, Controller controller) {
+        return jobType == JobType.component
+               ? Optional.empty()
+               : deployRevisionIn(jobType.zone(controller.system()).get());
     }
 
-    /** Returns the version a deployment to this zone should use for this application, or empty if we don't know */
-    private Optional<ApplicationRevision> currentDeployRevision(Zone zone) {
-        if (!deploying().isPresent()) {
-            return currentRevision(zone);
-        } else if (deploying().get() instanceof Change.VersionChange) {
-            return currentRevision(zone);
-        } else {
+    /** Returns the revision a new deployment to this zone should use for this application, or empty if we don't know */
+    private Optional<ApplicationRevision> deployRevisionIn(Zone zone) {
+        if (deploying().isPresent() && deploying().get() instanceof ApplicationChange)
             return ((Change.ApplicationChange) deploying().get()).revision();
-        }
+
+        return revisionIn(zone);
     }
 
-    /**
-     * Returns the current revision this application has, or if none; should use assuming no change,
-     * in the given zone. Empty if not known
-     */
-    private Optional<ApplicationRevision> currentRevision(Zone zone) {
-        Deployment currentDeployment = deployments().get(zone);
-        if (currentDeployment != null) { // Already deployed in this zone: Use that revision
-            return Optional.of(currentDeployment.revision());
-        }
-        return Optional.empty();
+    /** Returns the revision this application is or should be deployed with in the given zone, or empty if unknown. */
+    private Optional<ApplicationRevision> revisionIn(Zone zone) {
+        return Optional.ofNullable(deployments().get(zone)).map(Deployment::revision);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index bf5529c210f..c8eaa1bae16 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -172,7 +172,7 @@ public class ApplicationList {
      * Applications without any deployments are ordered first.
      */
     public ApplicationList byIncreasingDeployedVersion() {
-        return listOf(list.stream().sorted(Comparator.comparing(application -> application.deployedVersion().orElse(Version.emptyVersion))));
+        return listOf(list.stream().sorted(Comparator.comparing(application -> application.oldestDeployedVersion().orElse(Version.emptyVersion))));
     }
 
     /** Returns the subset of applications that are not currently upgrading */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
index bf067a39bbb..d9c22018d26 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
@@ -33,7 +33,7 @@ public abstract class Change {
 
         @Override
         public boolean blockedBy(DeploymentSpec deploymentSpec, Instant instant) {
-            return  ! deploymentSpec.canChangeRevisionAt(instant);
+            return ! deploymentSpec.canChangeRevisionAt(instant);
         }
 
         @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
index b5733703fd4..98f8c2a3d99 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/DeploymentJobs.java
@@ -162,7 +162,7 @@ public class DeploymentJobs {
     /** Job types that exist in the build system */
     public enum JobType {
 
-        component              ("component")                ,
+        component              ("component"                 ),
         systemTest             ("system-test"               , zone("test"   , "us-east-1"     ), zone(SystemName.cd, "test"   , "cd-us-central-1")),
         stagingTest            ("staging-test"              , zone("staging", "us-east-3"     ), zone(SystemName.cd, "staging", "cd-us-central-1")),
         productionCorpUsEast1  ("production-corp-us-east-1" , zone("prod"   , "corp-us-east-1")),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
index 1f7a61599b3..7c06ef27ce9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
@@ -46,6 +46,7 @@ public class DeploymentOrder {
 
     /** Returns a list of jobs to trigger after the given job */
     // TODO: This does too much - should just tell us the order, as advertised
+    // TODO: You're next!
     public List<JobType> nextAfter(JobType job, LockedApplication application) {
         if ( ! application.deploying().isPresent()) { // Change was cancelled
             return Collections.emptyList();
@@ -88,11 +89,6 @@ public class DeploymentOrder {
                 .collect(collectingAndThen(toList(), Collections::unmodifiableList));
     }
 
-    /** Returns whether the given job causes an application change */
-    public boolean givesNewRevision(JobType job) {
-        return job == JobType.component;
-    }
-
     /** Returns jobs for given deployment spec, in the order they are declared */
     public List<JobType> jobsFrom(DeploymentSpec deploymentSpec) {
         return deploymentSpec.steps().stream()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 97ac317d15b..7f11754a8f4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -2,7 +2,6 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.component.Version;
-import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.SystemName;
@@ -14,8 +13,8 @@ import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.LockedApplication;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Change.VersionChange;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
@@ -27,7 +26,6 @@ import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.ArrayList;
-import java.util.Comparator;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
@@ -67,7 +65,11 @@ public class DeploymentTrigger {
     
     /** Returns the time in the past before which jobs are at this moment considered unresponsive */
     public Instant jobTimeoutLimit() { return clock.instant().minus(jobTimeout); }
-    
+
+    public BuildSystem buildSystem() { return buildSystem; }
+
+    public DeploymentOrder deploymentOrder() { return order; }
+
     //--- Start of methods which triggers deployment jobs -------------------------
 
     /** 
@@ -83,7 +85,7 @@ public class DeploymentTrigger {
 
             // Handle successful starting and ending
             if (report.success()) {
-                if (order.givesNewRevision(report.jobType())) {
+                if (report.jobType() == JobType.component) {
                     if (acceptNewRevisionNow(application)) {
                         // Set this as the change we are doing, unless we are already pushing a platform change
                         if ( ! ( application.deploying().isPresent() &&
@@ -105,10 +107,10 @@ public class DeploymentTrigger {
             if (report.success())
                 application = trigger(order.nextAfter(report.jobType(), application), application,
                                       report.jobType().jobName() + " completed");
-            else if (isCapacityConstrained(report.jobType()) && shouldRetryOnOutOfCapacity(application, report.jobType()))
+            else if (retryBecauseOutOfCapacity(application, report.jobType()))
                 application = trigger(report.jobType(), application, true,
                                       "Retrying on out of capacity");
-            else if (shouldRetryNow(application, report.jobType()))
+            else if (retryBecauseNewFailure(application, report.jobType()))
                 application = trigger(report.jobType(), application, false,
                                       "Immediate retry on failure");
 
@@ -199,6 +201,7 @@ public class DeploymentTrigger {
             Version targetVersion = ((Change.VersionChange)change).version();
             if ( ! (targetVersion.equals(previous.lastSuccess().get().version())) )
                 return false; // version is outdated
+            // The below is checked again in allowedTriggering, right before actual triggering.
             if (next != null && isOnNewerVersionInProductionThan(targetVersion, application, next.type()))
                 return false; // Don't downgrade
         }
@@ -254,39 +257,24 @@ public class DeploymentTrigger {
 
     private ApplicationController applications() { return controller.applications(); }
 
-    private boolean isCapacityConstrained(JobType jobType) {
-        return jobType == JobType.stagingTest || jobType == JobType.systemTest;
-    }
-
     /** Retry immediately only if this job just started failing. Otherwise retry periodically */
-    private boolean shouldRetryNow(Application application, JobType jobType) {
+    private boolean retryBecauseNewFailure(Application application, JobType jobType) {
         JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
         return (jobStatus != null && jobStatus.firstFailing().get().at().isAfter(clock.instant().minus(Duration.ofSeconds(10))));
     }
 
     /** Decide whether to retry due to capacity restrictions */
-    private boolean shouldRetryOnOutOfCapacity(Application application, JobType jobType) {
-        Optional<JobError> outOfCapacityError = Optional.ofNullable(application.deploymentJobs().jobStatus().get(jobType))
-                .flatMap(JobStatus::jobError)
-                .filter(e -> e.equals(JobError.outOfCapacity));
-
-        if ( ! outOfCapacityError.isPresent()) return false;
-
+    private boolean retryBecauseOutOfCapacity(Application application, JobType jobType) {
+        JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
+        if (jobStatus == null || ! jobStatus.jobError().equals(Optional.of(JobError.outOfCapacity))) return false;
         // Retry the job if it failed recently
-        return application.deploymentJobs().jobStatus().get(jobType).firstFailing().get().at()
-                .isAfter(clock.instant().minus(Duration.ofMinutes(15)));
+        return jobStatus.firstFailing().get().at().isAfter(clock.instant().minus(Duration.ofMinutes(15)));
     }
 
     /** Returns whether the given job type should be triggered according to deployment spec */
-    private boolean deploysTo(Application application, JobType jobType) {
-        Optional<Zone> zone = jobType.zone(controller.system());
-        if (zone.isPresent() && jobType.isProduction()) {
-            // Skip triggering of jobs for zones where the application should not be deployed
-            if ( ! application.deploymentSpec().includes(jobType.environment(), Optional.of(zone.get().region()))) {
-                return false;
-            }
-        }
-        return true;
+    private boolean hasJob(JobType jobType, Application application) {
+        if ( ! jobType.isProduction()) return true; // Deployment spec only determines this for production jobs.
+        return application.deploymentSpec().includes(jobType.environment(), jobType.region(controller.system()));
     }
 
     /**
@@ -345,16 +333,18 @@ public class DeploymentTrigger {
         //       by instead basing the decision on what is currently deployed in the zone. However,
         //       this leads to some additional corner cases, and the possibility of blocking an application
         //       fix to a version upgrade, so not doing it now
-        if (jobType.isProduction() && application.deployingBlocked(clock.instant())) return false;
+
+        if (jobType.isProduction() && application.deploying().isPresent() &&
+            application.deploying().get().blockedBy(application.deploymentSpec(), clock.instant())) return false;
+
+        if (application.deploying().isPresent() && application.deploying().get() instanceof VersionChange &&
+            isOnNewerVersionInProductionThan(((VersionChange) application.deploying().get()).version(), application, jobType)) return false;
+
         if (application.deploymentJobs().isRunning(jobType, jobTimeoutLimit())) return false;
-        if  ( ! deploysTo(application, jobType)) return false;
+        if  ( ! hasJob(jobType, application)) return false;
         // Ignore applications that are not associated with a project
         if ( ! application.deploymentJobs().projectId().isPresent()) return false;
-        if (application.deploying().isPresent() && application.deploying().get() instanceof Change.VersionChange) {
-            Version targetVersion = ((Change.VersionChange)application.deploying().get()).version();
-            if (isOnNewerVersionInProductionThan(targetVersion, application, jobType)) return false; // Don't downgrade
-        }
-        
+
         return true;
     }
     
@@ -373,7 +363,7 @@ public class DeploymentTrigger {
      * downgrade production nodes which we are not guaranteed to support.
      */
     private boolean isOnNewerVersionInProductionThan(Version version, Application application, JobType job) {
-        if ( ! isProduction(job)) return false;
+        if ( ! job.isProduction()) return false;
         Optional<Zone> zone = job.zone(controller.system());
         if ( ! zone.isPresent()) return false;
         Deployment existingDeployment = application.deployments().get(zone.get());
@@ -381,24 +371,17 @@ public class DeploymentTrigger {
         return existingDeployment.version().isAfter(version);
     }
     
-    private boolean isProduction(JobType job) {
-        Optional<Zone> zone = job.zone(controller.system());
-        if ( ! zone.isPresent()) return false; // arbitrary
-        return zone.get().environment() == Environment.prod;
-    }
-    
     private boolean acceptNewRevisionNow(LockedApplication application) {
         if ( ! application.deploying().isPresent()) return true;
-        if ( application.deploying().get() instanceof Change.ApplicationChange) return true; // more changes are ok
 
-        if ( application.deploymentJobs().hasFailures()) return true; // allow changes to fix upgrade problems
-        if ( application.isBlocked(clock.instant())) return true; // allow testing changes while upgrade blocked (debatable)
+        if (application.deploying().get() instanceof Change.ApplicationChange) return true; // more changes are ok
+
+        if (application.deploymentJobs().hasFailures()) return true; // allow changes to fix upgrade problems
+
+        if (application.isBlocked(clock.instant())) return true; // allow testing changes while upgrade blocked (debatable)
+
         // Otherwise, the application is currently upgrading, without failures, and we should wait with the revision.
         return false;
     }
     
-    public BuildSystem buildSystem() { return buildSystem; }
-
-    public DeploymentOrder deploymentOrder() { return order; }
-
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 1bb116809f7..56d7406b080 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -373,7 +373,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
 
         // Compile version. The version that should be used when building an application
-        object.setString("compileVersion", application.compileVersion(controller).toFullString());
+        object.setString("compileVersion", application.oldestDeployedVersion().orElse(controller.systemVersion()).toFullString());
 
         // Rotations
         Cursor globalRotationsArray = object.setArray("globalRotations");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 5f4d40ed2d8..ef3f0658f2b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -219,7 +219,7 @@ public class ControllerTest {
         tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
 
         app1 = applications.require(app1.id());
-        assertEquals("First deployment gets system version", systemVersion, app1.deployedVersion().get());
+        assertEquals("First deployment gets system version", systemVersion, app1.oldestDeployedVersion().get());
         assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get());
 
         // Unexpected deployment
@@ -242,13 +242,13 @@ public class ControllerTest {
         tester.deployAndNotify(app1, applicationPackage, true, productionUsWest1);
 
         app1 = applications.require(app1.id());
-        assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
+        assertEquals("Application change preserves version", systemVersion, app1.oldestDeployedVersion().get());
         assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get());
 
         // A deployment to the new region gets the same version
         tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
         app1 = applications.require(app1.id());
-        assertEquals("Application change preserves version", systemVersion, app1.deployedVersion().get());
+        assertEquals("Application change preserves version", systemVersion, app1.oldestDeployedVersion().get());
         assertEquals(systemVersion, tester.configServer().lastPrepareVersion().get());
         assertFalse("Change deployed", app1.deploying().isPresent());
 
@@ -261,7 +261,7 @@ public class ControllerTest {
         tester.deployAndNotify(app1, applicationPackage, true, productionUsEast3);
 
         app1 = applications.require(app1.id());
-        assertEquals("Version upgrade changes version", newSystemVersion, app1.deployedVersion().get());
+        assertEquals("Version upgrade changes version", newSystemVersion, app1.oldestDeployedVersion().get());
         assertEquals(newSystemVersion, tester.configServer().lastPrepareVersion().get());
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index 64082adc1c0..8839f6a5a18 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -607,7 +607,7 @@ public class UpgraderTest {
         Application default3 = tester.createAndDeploy("default3", 6, "default");
         Application default4 = tester.createAndDeploy("default4", 7, "default");
         
-        assertEquals(version, default0.deployedVersion().get());
+        assertEquals(version, default0.oldestDeployedVersion().get());
 
         // New version is released
         version = Version.fromString("5.1");
@@ -664,10 +664,10 @@ public class UpgraderTest {
         tester.completeUpgrade(default2, version, "default");
         tester.completeUpgrade(default3, version, "default");
 
-        assertEquals(version, tester.application(default0.id()).deployedVersion().get());
-        assertEquals(version, tester.application(default1.id()).deployedVersion().get());
-        assertEquals(version, tester.application(default2.id()).deployedVersion().get());
-        assertEquals(version, tester.application(default3.id()).deployedVersion().get());
+        assertEquals(version, tester.application(default0.id()).oldestDeployedVersion().get());
+        assertEquals(version, tester.application(default1.id()).oldestDeployedVersion().get());
+        assertEquals(version, tester.application(default2.id()).oldestDeployedVersion().get());
+        assertEquals(version, tester.application(default3.id()).oldestDeployedVersion().get());
     }
 
     @Test