Handle incocmpatible versions for dev/perf deployments

6 files changed, 68 insertions, 21 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index a4aac26b973..61fce68b5d5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -875,11 +875,9 @@ public class ApplicationController {
     /** Returns the latest known version within the given major, which is not newer than the system version. */
     public Optional<Version> lastCompatibleVersion(int targetMajorVersion) {
         VersionStatus versions = controller.readVersionStatus();
-        Version systemVersion = controller.systemVersion(versions);
-        return versions.versions().stream()
+        return versions.deployableVersions().stream()
                        .map(VespaVersion::versionNumber)
                        .filter(version -> version.getMajor() == targetMajorVersion)
-                       .filter(version -> ! version.isAfter(systemVersion))
                        .max(naturalOrder());
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index b0e51c8fe9d..03dfa0134cf 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -142,9 +142,8 @@ public class DeploymentTrigger {
             return List.of(systemVersion);
 
         VespaVersion.Confidence target = policy == DeploymentSpec.UpgradePolicy.defaultPolicy ? VespaVersion.Confidence.normal : VespaVersion.Confidence.high;
-        return versions.versions().stream()
-                       .filter(version ->    ! version.versionNumber().isAfter(systemVersion)
-                                          &&   version.confidence().equalOrHigherThan(target))
+        return versions.deployableVersions().stream()
+                       .filter(version -> version.confidence().equalOrHigherThan(target))
                        .map(VespaVersion::versionNumber)
                        .sorted(reverseOrder())
                        .collect(Collectors.toList());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index abd5ca8891a..6413d9e0def 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -2,7 +2,9 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableSortedMap;
+import com.yahoo.collections.Iterables;
 import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
@@ -26,13 +28,17 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageDiff;
 import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.ArrayDeque;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
+import java.util.Deque;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -49,6 +55,7 @@ import java.util.function.UnaryOperator;
 import java.util.logging.Level;
 import java.util.stream.Stream;
 
+import static com.yahoo.collections.Iterables.reversed;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.reset;
 import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
@@ -533,8 +540,10 @@ public class JobController {
         lastRun.filter(run -> ! run.hasEnded()).ifPresent(run -> abortAndWait(run.id()));
 
         long build = 1 + lastRun.map(run -> run.versions().targetApplication().buildNumber().orElse(0)).orElse(0L);
-        ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(), Optional.empty(),
-                Optional.empty(), Optional.empty(), Optional.empty(), true, Optional.empty());
+        ApplicationVersion version = ApplicationVersion.from(Optional.empty(), build, Optional.empty(),
+                                                             applicationPackage.compileVersion(),
+                                                             Optional.empty(), Optional.empty(),
+                                                             Optional.empty(), true, Optional.empty());
 
         byte[] diff = lastRun.map(run -> run.versions().targetApplication())
                 .map(prevVersion -> ApplicationPackageDiff.diff(new ApplicationPackage(controller.applications().applicationStore().get(deploymentId, prevVersion)), applicationPackage))
@@ -542,16 +551,10 @@ public class JobController {
 
         controller.applications().lockApplicationOrThrow(TenantAndApplicationId.from(id), application -> {
             controller.applications().applicationStore().putDev(deploymentId, version, applicationPackage.zippedContent(), diff);
+            Version targetPlatform = platform.orElseGet(() -> findTargetPlatform(applicationPackage, lastRun));
             start(id,
                   type,
-                  new Versions(platform.orElse(applicationPackage.deploymentSpec().majorVersion()
-                                                                 .flatMap(controller.applications()::lastCompatibleVersion)
-                                                                 .or(() -> lastRun.map(run -> run.versions().targetPlatform())
-                                                                                  .filter(controller.readVersionStatus()::isActive))
-                                                                 .orElseGet(controller::readSystemVersion)),
-                               version,
-                               lastRun.map(run -> run.versions().targetPlatform()),
-                               lastRun.map(run -> run.versions().targetApplication())),
+                  new Versions(targetPlatform, version, lastRun.map(run -> run.versions().targetPlatform()), lastRun.map(run -> run.versions().targetApplication())),
                   false,
                   dryRun ? JobProfile.developmentDryRun : JobProfile.development,
                   Optional.empty());
@@ -562,6 +565,31 @@ public class JobController {
         });
     }
 
+    private Version findTargetPlatform(ApplicationPackage applicationPackage, Optional<Run> lastRun) {
+        Optional<Integer> major = applicationPackage.deploymentSpec().majorVersion();
+        if (major.isPresent())
+            return controller.applications().lastCompatibleVersion(major.get())
+                             .orElseThrow(() -> new IllegalArgumentException("major " + major.get() + " specified in deployment.xml, " +
+                                                                             "but no version on this major was found"));
+
+        // Prefer previous platform if possible.
+        VersionStatus versionStatus = controller.readVersionStatus();
+        VersionCompatibility compatibility = controller.applications().versionCompatibility();
+        Optional<Version> target = lastRun.map(run -> run.versions().targetPlatform()).filter(versionStatus::isActive);
+        if (target.isPresent() && compatibility.accept(target.get(), applicationPackage.compileVersion().orElse(target.get())))
+            return target.get();
+
+        // Otherwise, use newest, compatible version.
+        for (VespaVersion platform : reversed(versionStatus.deployableVersions()))
+            if (compatibility.accept(platform.versionNumber(), applicationPackage.compileVersion().orElse(platform.versionNumber())))
+                return platform.versionNumber();
+
+        throw new IllegalArgumentException("no suitable platform version found" +
+                                           applicationPackage.compileVersion()
+                                                             .map(version -> " for package compiled against " + version)
+                                                             .orElse(""));
+    }
+
     /** Aborts a run and waits for it complete. */
     private void abortAndWait(RunId id) {
         abort(id, "replaced by new deployment");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
index bf3b9b90f66..57a4a042203 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentUpgrader.java
@@ -10,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.deployment.Run;
 import com.yahoo.vespa.hosted.controller.deployment.Versions;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
 
 import java.time.Duration;
@@ -18,6 +19,8 @@ import java.util.Optional;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.Level;
 
+import static com.yahoo.collections.Iterables.reversed;
+
 /**
  * Upgrades instances in manually deployed zones to the system version, at a convenient time.
  * 
@@ -33,22 +36,32 @@ public class DeploymentUpgrader extends ControllerMaintainer {
     protected double maintain() {
         AtomicInteger attempts = new AtomicInteger();
         AtomicInteger failures = new AtomicInteger();
-        Version systemVersion = controller().readSystemVersion();
+
+        Version targetPlatform = null; // Upgrade to the newest non-broken, deployable version.
+        for (VespaVersion platform : controller().readVersionStatus().deployableVersions())
+            if (platform.confidence().equalOrHigherThan(VespaVersion.Confidence.low))
+                targetPlatform = platform.versionNumber();
+
+        if (targetPlatform == null)
+            return 0;
 
         for (Application application : controller().applications().readable())
             for (Instance instance : application.instances().values())
                 for (Deployment deployment : instance.deployments().values())
                     try {
-                        attempts.incrementAndGet();
                         JobId job = new JobId(instance.id(), JobType.from(controller().system(), deployment.zone()).get());
                         if ( ! deployment.zone().environment().isManuallyDeployed()) continue;
 
                         Run last = controller().jobController().last(job).get();
-                        Versions target = new Versions(systemVersion, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication()));
+                        Versions target = new Versions(targetPlatform, last.versions().targetApplication(), Optional.of(last.versions().targetPlatform()), Optional.of(last.versions().targetApplication()));
+                        if (last.versions().targetApplication().compileVersion()
+                                .map(version -> controller().applications().versionCompatibility().refuse(version, target.targetPlatform()))
+                                .orElse(false)) continue;
                         if ( ! deployment.version().isBefore(target.targetPlatform())) continue;
                         if ( ! isLikelyNightFor(job)) continue;
 
                         log.log(Level.FINE, "Upgrading deployment of " + instance.id() + " in " + deployment.zone());
+                        attempts.incrementAndGet();
                         controller().jobController().start(instance.id(), JobType.from(controller().system(), deployment.zone()).get(), target, true, Optional.of("automated upgrade"));
                     } catch (Exception e) {
                         failures.incrementAndGet();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index f723d8b5134..545f38bf6df 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -25,10 +25,8 @@ import java.util.Random;
 import java.util.function.UnaryOperator;
 import java.util.logging.Level;
 import java.util.logging.Logger;
-import java.util.stream.Collectors;
 
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM;
-import static java.util.Comparator.reverseOrder;
 
 /**
  * Maintenance job which schedules applications for Vespa version upgrade
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
index 238ab0b09fa..3fa440e694e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java
@@ -69,6 +69,17 @@ public class VersionStatus {
      * Calling this is free, but the returned status is slightly out of date.
      */
     public List<VespaVersion> versions() { return versions; }
+
+    /** Lists all currently active Vespa versions, from lowest to highest number, which are not newer than the system version. */
+    public List<VespaVersion> deployableVersions() {
+        List<VespaVersion> deployable = new ArrayList<>();
+        for (VespaVersion version : versions) {
+            deployable.add(version);
+            if (version.isSystemVersion())
+                return deployable;
+        }
+        return List.of();
+    }
     
     /** Returns the given version, or null if it is not present */
     public VespaVersion version(Version version) {