Merge pull request #21680 from vespa-engine/jonmv/major-upgrade-orchestration

Jonmv/major upgrade orchestration

16 files changed, 405 insertions, 64 deletions

diff --git a/component/abi-spec.json b/component/abi-spec.json
index 4e89b0e717c..d990a9077b4 100644
--- a/component/abi-spec.json
+++ b/component/abi-spec.json
@@ -158,6 +158,19 @@
       "public static final com.yahoo.component.Version emptyVersion"
     ]
   },
+  "com.yahoo.component.VersionCompatibility": {
+    "superClass": "java.lang.Object",
+    "interfaces": [],
+    "attributes": [
+      "public"
+    ],
+    "methods": [
+      "public static com.yahoo.component.VersionCompatibility fromVersionList(java.util.List)",
+      "public boolean accept(com.yahoo.component.Version, com.yahoo.component.Version)",
+      "public boolean refuse(com.yahoo.component.Version, com.yahoo.component.Version)"
+    ],
+    "fields": []
+  },
   "com.yahoo.component.VersionSpecification": {
     "superClass": "java.lang.Object",
     "interfaces": [
diff --git a/component/src/main/java/com/yahoo/component/VersionCompatibility.java b/component/src/main/java/com/yahoo/component/VersionCompatibility.java
new file mode 100644
index 00000000000..fb9f863f5f0
--- /dev/null
+++ b/component/src/main/java/com/yahoo/component/VersionCompatibility.java
@@ -0,0 +1,109 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.component;
+
+import java.util.List;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+/**
+ * Logic for what platform and compile versions are incompatible.
+ *
+ * This class is instantiated with a list of versions which are only compatible with other versions
+ * that are not older than themselves. Each entry in this list is represented by a string, and any
+ * major, minor or micro with the value @{code *} means all versions are such boundaries.
+ * Versions A, B are incompatible iff. there exists an incompatibility boundary X such that
+ * (A ≥ X) ≠ (B ≥ X).
+ *
+ * @author jonmv
+ */
+public class VersionCompatibility {
+
+    private final Node root;
+
+    private VersionCompatibility(Node root) {
+        this.root = root;
+    }
+
+    public static VersionCompatibility fromVersionList(List<String> versions) {
+        Node root = new Node();
+        for (String spec : versions) {
+            String[] parts = spec.split("\\.");
+            if (parts.length < 1 || parts.length > 3)
+                throw new IllegalArgumentException("Each spec must have 1 to 3 parts, but found '" + spec + "'");
+
+            boolean wildcard = false;
+            Node node = root;
+            for (int i = 0; i < 3; i++) {
+                String part = i < parts.length ? parts[i] : wildcard ? null : "0";
+                if (wildcard && part != null && ! part.equals("*"))
+                    throw new IllegalArgumentException("Wildcard parts may only have wildcard children, but found '" + spec + "'");
+
+                if ("*".equals(part)) {
+                    wildcard = true;
+                    if (node.children.isEmpty())
+                        node = node.children.computeIfAbsent(-1, __ -> new Node());
+                    else
+                        throw new IllegalArgumentException("Wildcards may not have siblings, but got: " + versions);
+                }
+                else if (part != null) {
+                    int number = Integer.parseInt(part);
+                    if (number < 0)
+                        throw new IllegalArgumentException("Version parts must be non-negative, but found '" + spec + "'");
+                    if (node.children.containsKey(-1))
+                        throw new IllegalArgumentException("Wildcards may not have siblings, but got: " + versions);
+                    if (i < 2)
+                        node = node.children.computeIfAbsent(number, __ -> new Node());
+                    else if (node.children.put(number, new Node()) != null)
+                        throw new IllegalArgumentException("Duplicate element '" + spec + "'");
+                }
+            }
+        }
+        return new VersionCompatibility(root);
+    }
+
+    public boolean accept(Version first, Version second) {
+        return ! refuse(first, second);
+    }
+
+    public boolean refuse(Version first, Version second) {
+        if (first.compareTo(second) > 0)
+            return refuse(second, first);
+
+        if (first.compareTo(second) == 0)
+            return false;
+
+        return refuse(new int[]{  first.getMajor(),  first.getMinor(),  first.getMicro() },
+                      new int[]{ second.getMajor(), second.getMinor(), second.getMicro() },
+                      0, root, root);
+    }
+
+    private boolean refuse(int[] first, int[] second, int i, Node left, Node right) {
+        if (left == null && right == null) return false;
+        if (i == 3) return right != null;
+        int u = first[i], v = second[i];
+        if (left == right) {
+            Node wildcard = left.children.get(-1);
+            if (wildcard != null)
+                return u != v || refuse(first, second, i + 1, wildcard, wildcard);
+
+            if ( ! left.children.tailMap(u, false).headMap(v, false).isEmpty())
+                return true;
+
+            return refuse(first, second, i + 1, left.children.get(u), left.children.get(v));
+        }
+        if (left != null && (left.children.containsKey(-1) || ! left.children.tailMap(u, false).isEmpty()))
+            return true;
+
+        if (right != null && (right.children.containsKey(-1) || ! right.children.headMap(v, false).isEmpty()))
+            return true;
+
+        return refuse(first, second, i + 1, left == null ? null : left.children.get(u), right == null ? null : right.children.get(v));
+    }
+
+    private static class Node {
+
+        final NavigableMap<Integer, Node> children = new TreeMap<>();
+
+    }
+
+}
diff --git a/component/src/test/java/com/yahoo/component/VersionCompatibilityTest.java b/component/src/test/java/com/yahoo/component/VersionCompatibilityTest.java
new file mode 100644
index 00000000000..c4f0e0f7439
--- /dev/null
+++ b/component/src/test/java/com/yahoo/component/VersionCompatibilityTest.java
@@ -0,0 +1,87 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.component;
+
+import org.junit.Test;
+
+import java.util.List;
+import java.util.Locale;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+/**
+ * @author jonmv
+ */
+public class VersionCompatibilityTest {
+
+    @Test
+    public void testNoIncompatibilities() {
+        List<String> versions = List.of();
+        VersionCompatibility compatibility = VersionCompatibility.fromVersionList(versions);
+        assertTrue(compatibility.accept(new Version(0, 0, 0), new Version(0, 0, 0)));
+        assertTrue(compatibility.accept(new Version(0, 0, 0), new Version(0, 0, 1)));
+        assertTrue(compatibility.accept(new Version(0, 0, 0), new Version(0, 1, 0)));
+        assertTrue(compatibility.accept(new Version(0, 0, 0), new Version(1, 0, 0)));
+    }
+
+    @Test
+    public void testValidIncompatibilities() {
+        List<String> versions = List.of("1.2.*", "2", "3.*", "4.0.0", "4.0.1", "4.0.3", "4.1.0", "5");
+        VersionCompatibility compatibility = VersionCompatibility.fromVersionList(versions);
+        assertTrue (compatibility.accept(new Version(0, 0, 0), new Version(0, 0, 0)));
+        assertTrue (compatibility.accept(new Version(0, 0, 0), new Version(1, 1, 1)));
+        assertFalse(compatibility.accept(new Version(0, 0, 0), new Version(1, 2, 3)));
+        assertFalse(compatibility.accept(new Version(1, 1, 0), new Version(1, 2, 0)));
+        assertFalse(compatibility.accept(new Version(1, 2, 1), new Version(1, 2, 0)));
+        assertFalse(compatibility.accept(new Version(1, 1, 0), new Version(1, 3, 0)));
+        assertTrue (compatibility.accept(new Version(1, 2, 3), new Version(1, 2, 3)));
+        assertTrue (compatibility.accept(new Version(1, 3, 0), new Version(1, 9, 9)));
+        assertFalse(compatibility.accept(new Version(1, 3, 0), new Version(2, 0, 0)));
+        assertTrue (compatibility.accept(new Version(2, 0, 0), new Version(2, 2, 2)));
+        assertFalse(compatibility.accept(new Version(2, 0, 0), new Version(3, 0, 0)));
+        assertTrue (compatibility.accept(new Version(3, 0, 0), new Version(3, 0, 0)));
+        assertFalse(compatibility.accept(new Version(3, 0, 0), new Version(3, 1, 0)));
+        assertTrue (compatibility.accept(new Version(3, 0, 0), new Version(3, 0, 1)));
+        assertFalse(compatibility.accept(new Version(3, 0, 0), new Version(4, 0, 0)));
+        assertFalse(compatibility.accept(new Version(4, 0, 0), new Version(4, 0, 1)));
+        assertTrue (compatibility.accept(new Version(4, 0, 1), new Version(4, 0, 2)));
+        assertFalse(compatibility.accept(new Version(4, 0, 2), new Version(4, 0, 3)));
+        assertFalse(compatibility.accept(new Version(4, 0, 3), new Version(4, 1, 0)));
+        assertFalse(compatibility.accept(new Version(4, 1, 0), new Version(5, 0, 0)));
+        assertTrue (compatibility.accept(new Version(5, 0, 0), new Version(6, 0, 0)));
+        assertFalse(compatibility.accept(new Version(0, 0, 0), new Version(2, 0, 0)));
+        assertFalse(compatibility.accept(new Version(0, 0, 0), new Version(6, 0, 0)));
+    }
+
+    @Test
+    public void testIllegalIncompatibilities() {
+        assertThrows(List.of("1", "*"),         IllegalArgumentException.class, "may not have siblings");
+        assertThrows(List.of("*", "*.*"),       IllegalArgumentException.class, "may not have siblings");
+        assertThrows(List.of("*", "*"),         IllegalArgumentException.class, "may not have siblings");
+        assertThrows(List.of("*.1"),            IllegalArgumentException.class, "may only have wildcard children");
+        assertThrows(List.of("0", "0"),         IllegalArgumentException.class, "duplicate element");
+        assertThrows(List.of("0", "0.0.0"),     IllegalArgumentException.class, "duplicate element");
+        assertThrows(List.of("0.0.0", "0.0.0"), IllegalArgumentException.class, "duplicate element");
+        assertThrows(List.of("."),              IllegalArgumentException.class, "1 to 3 parts");
+        assertThrows(List.of("0.0.0.0"),        IllegalArgumentException.class, "1 to 3 parts");
+        assertThrows(List.of("-1"),             IllegalArgumentException.class, "must be non-negative");
+        assertThrows(List.of(""),               NumberFormatException.class,    "input string: \"\"");
+        assertThrows(List.of("x"),              NumberFormatException.class,    "input string: \"x\"");
+    }
+
+    static void assertThrows(List<String> spec, Class<? extends IllegalArgumentException> clazz, String fragment) {
+        IllegalArgumentException thrown = null;
+        try {
+            VersionCompatibility.fromVersionList(spec);
+        }
+        catch (IllegalArgumentException e) {
+            if (clazz.isInstance(e) && e.getMessage().toLowerCase(Locale.ROOT).contains(fragment.toLowerCase(Locale.ROOT)))
+                return;
+            thrown = e;
+        }
+        fail("Should fail with " + clazz.getSimpleName() + " containing '" + fragment + "' in its message, but got " +
+             (thrown == null ? "no exception" : "'" + thrown + "'"));
+    }
+
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
index 5e9e1e8bf33..02b48c9d626 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/application/TenantApplications.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.application;
 
 import com.yahoo.cloud.config.ConfigserverConfig;
 import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.concurrent.StripedExecutor;
 import com.yahoo.config.FileReference;
 import com.yahoo.config.provision.ApplicationId;
@@ -74,7 +75,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica
     private final Clock clock;
     private final TenantFileSystemDirs tenantFileSystemDirs;
     private final ConfigserverConfig configserverConfig;
-    private final ListFlag<Integer> incompatibleMajorVersions;
+    private final ListFlag<String> incompatibleVersions;
 
     public TenantApplications(TenantName tenant, Curator curator, StripedExecutor<TenantName> zkWatcherExecutor,
                               ExecutorService zkCacheExecutor, Metrics metrics, ReloadListener reloadListener,
@@ -95,7 +96,7 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica
         this.tenantFileSystemDirs = tenantFileSystemDirs;
         this.clock = clock;
         this.configserverConfig = configserverConfig;
-        this.incompatibleMajorVersions = PermanentFlags.INCOMPATIBLE_MAJOR_VERSIONS.bindTo(flagSource);
+        this.incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource);
     }
 
     /** The curator backed ZK storage of this. */
@@ -392,8 +393,8 @@ public class TenantApplications implements RequestHandler, HostValidator<Applica
         if (vespaVersion.isEmpty()) return true;
         Version wantedVersion = applicationMapper.getForVersion(application, Optional.empty(), clock.instant())
                                                  .getModel().wantedNodeVersion();
-        boolean compatibleMajor = ! incompatibleMajorVersions.value().contains(wantedVersion.getMajor());
-        return compatibleMajor || vespaVersion.get().getMajor() == wantedVersion.getMajor();
+        return VersionCompatibility.fromVersionList(incompatibleVersions.value())
+                                   .accept(vespaVersion.get(), wantedVersion);
     }
 
     @Override
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java
index c95c95750a1..93637536182 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/application/TenantApplicationsTest.java
@@ -186,7 +186,7 @@ public class TenantApplicationsTest {
         applications.activateApplication(createSet(app1, deployedVersion1), 1);
         assertTrue("New major is compatible", applications.compatibleWith(Optional.of(nodeVersion0), app1));
 
-        flagSource.withListFlag(PermanentFlags.INCOMPATIBLE_MAJOR_VERSIONS.id(), List.of(8), Integer.class);
+        flagSource.withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class);
         Version deployedVersion2 = Version.fromString("8.1");
         applications.activateApplication(createSet(app1, deployedVersion2), 1);
         assertFalse("New major is incompatible", applications.compatibleWith(Optional.of(nodeVersion0), app1));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 5c3c43461dc..a4aac26b973 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller;
 
 import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
@@ -20,6 +21,7 @@ import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
+import com.yahoo.vespa.flags.ListFlag;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
@@ -123,6 +125,7 @@ public class ApplicationController {
     private final ApplicationPackageValidator applicationPackageValidator;
     private final EndpointCertificates endpointCertificates;
     private final StringFlag dockerImageRepoFlag;
+    private final ListFlag<String> incompatibleVersions;
     private final BillingController billingController;
 
     ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock,
@@ -137,6 +140,7 @@ public class ApplicationController {
         artifactRepository = controller.serviceRegistry().artifactRepository();
         applicationStore = controller.serviceRegistry().applicationStore();
         dockerImageRepoFlag = PermanentFlags.DOCKER_IMAGE_REPO.bindTo(flagSource);
+        incompatibleVersions = PermanentFlags.INCOMPATIBLE_VERSIONS.bindTo(flagSource);
         deploymentTrigger = new DeploymentTrigger(controller, clock);
         applicationPackageValidator = new ApplicationPackageValidator(controller);
         endpointCertificates = new EndpointCertificates(controller,
@@ -751,6 +755,10 @@ public class ApplicationController {
         return curator.lockForDeployment(application, zone);
     }
 
+    public VersionCompatibility versionCompatibility() {
+        return VersionCompatibility.fromVersionList(incompatibleVersions.value());
+    }
+
     /**
      * Verifies that the application can be deployed to the tenant, following these rules:
      *
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
index 91e80eb8a30..ce3cae08d36 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/InstanceList.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application;
 
 import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.InstanceName;
@@ -34,12 +35,23 @@ public class InstanceList extends AbstractFilteringList<ApplicationId, InstanceL
     }
 
     /**
-     * Returns the subset of instances that aren't pinned to an an earlier major version than the given one.
+     * Returns the subset of instances where all production deployments are compatible with the given version.
+     *
+     * @param platform the version which applications returned are compatible with
+     */
+    public InstanceList compatibleWithPlatform(Version platform, VersionCompatibility compatibility) {
+        return matching(id -> instance(id).productionDeployments().values().stream()
+                                          .flatMap(deployment -> deployment.applicationVersion().compileVersion().stream())
+                                          .noneMatch(version -> compatibility.refuse(platform, version)));
+    }
+
+    /**
+     * Returns the subset of instances that aren't pinned to an earlier major version than the given one.
      *
      * @param targetMajorVersion the target major version which applications returned allows upgrading to
      * @param defaultMajorVersion the default major version to assume for applications not specifying one
      */
-    public InstanceList allowMajorVersion(int targetMajorVersion, int defaultMajorVersion) {
+    public InstanceList allowingMajorVersion(int targetMajorVersion, int defaultMajorVersion) {
         return matching(id -> targetMajorVersion <= application(id).deploymentSpec().majorVersion()
                                                                    .orElse(application(id).majorVersion()
                                                                                           .orElse(defaultMajorVersion)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
index be38ce0c245..f94b9f32088 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentStatus.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest;
@@ -87,16 +88,18 @@ public class DeploymentStatus {
     private final JobList allJobs;
     private final SystemName system;
     private final Version systemVersion;
+    private final VersionCompatibility versionCompatibility;
     private final Instant now;
     private final Map<JobId, StepStatus> jobSteps;
     private final List<StepStatus> allSteps;
 
     public DeploymentStatus(Application application, Map<JobId, JobStatus> allJobs, SystemName system,
-                            Version systemVersion, Instant now) {
+                            Version systemVersion, VersionCompatibility versionCompatibility, Instant now) {
         this.application = requireNonNull(application);
         this.allJobs = JobList.from(allJobs.values());
         this.system = requireNonNull(system);
         this.systemVersion = requireNonNull(systemVersion);
+        this.versionCompatibility = versionCompatibility;
         this.now = requireNonNull(now);
         List<StepStatus> allSteps = new ArrayList<>();
         this.jobSteps = jobDependencies(application.deploymentSpec(), allSteps);
@@ -308,42 +311,54 @@ public class DeploymentStatus {
     private Map<JobId, List<Job>> productionJobs(InstanceName instance, Change change, boolean assumeUpgradesSucceed) {
         Map<JobId, List<Job>> jobs = new LinkedHashMap<>();
         jobSteps.forEach((job, step) -> {
+            if ( ! job.application().instance().equals(instance) || ! job.type().isProduction())
+                return;
+
+            // Signal strict completion criterion by depending on job itself.
+            if (step.completedAt(change, Optional.of(job)).isPresent())
+                return;
+
             // When computing eager test jobs for outstanding changes, assume current change completes successfully.
             Optional<Deployment> deployment = deploymentFor(job);
             Optional<Version> existingPlatform = deployment.map(Deployment::version);
             Optional<ApplicationVersion> existingApplication = deployment.map(Deployment::applicationVersion);
+            boolean deployingCompatibilityChange =    areIncompatible(existingPlatform, change.application())
+                                                   || areIncompatible(change.platform(), existingApplication);
             if (assumeUpgradesSucceed) {
+                if (deployingCompatibilityChange) // No eager tests for this.
+                    return;
+
                 Change currentChange = application.require(instance).change();
                 Versions target = Versions.from(currentChange, application, deployment, systemVersion);
                 existingPlatform = Optional.of(target.targetPlatform());
                 existingApplication = Optional.of(target.targetApplication());
             }
-            if (job.application().instance().equals(instance) && job.type().isProduction()) {
-                List<Job> toRun = new ArrayList<>();
-                List<Change> changes = changes(job, step, change);
-                if (changes.isEmpty()) return;
-                for (Change partial : changes) {
-                    Job jobToRun = new Job(job.type(),
-                                           Versions.from(partial, application, existingPlatform, existingApplication, systemVersion),
-                                           step.readyAt(partial, Optional.of(job)),
-                                           partial);
-                    toRun.add(jobToRun);
-                    // Assume first partial change is applied before the second.
-                    existingPlatform = Optional.of(jobToRun.versions.targetPlatform());
-                    existingApplication = Optional.of(jobToRun.versions.targetApplication());
-                }
-                jobs.put(job, toRun);
+            List<Job> toRun = new ArrayList<>();
+            List<Change> changes = deployingCompatibilityChange ? List.of(change) : changes(job, step, change);
+            if (changes.isEmpty()) return;
+            for (Change partial : changes) {
+                Job jobToRun = new Job(job.type(),
+                                       Versions.from(partial, application, existingPlatform, existingApplication, systemVersion),
+                                       step.readyAt(partial, Optional.of(job)),
+                                       partial);
+                toRun.add(jobToRun);
+                // Assume first partial change is applied before the second.
+                existingPlatform = Optional.of(jobToRun.versions.targetPlatform());
+                existingApplication = Optional.of(jobToRun.versions.targetApplication());
             }
+            jobs.put(job, toRun);
         });
         return jobs;
     }
 
+    private boolean areIncompatible(Optional<Version> platform, Optional<ApplicationVersion> application) {
+        return    platform.isPresent()
+               && application.flatMap(ApplicationVersion::compileVersion).isPresent()
+               && versionCompatibility.refuse(platform.get(), application.get().compileVersion().get());
+    }
+
     /** Changes to deploy with the given job, possibly split in two steps. */
     private List<Change> changes(JobId job, StepStatus step, Change change) {
-        // Signal strict completion criterion by depending on job itself.
-        if (step.completedAt(change, Optional.of(job)).isPresent())
-            return List.of();
-
         if (change.platform().isEmpty() || change.application().isEmpty() || change.isPinned())
             return List.of(change);
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index aeaa821745b..b0e51c8fe9d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -1,6 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.deployment;
 
+import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
@@ -19,6 +21,8 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
+import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -37,6 +41,7 @@ import java.util.logging.Logger;
 import java.util.stream.Collectors;
 
 import static java.util.Comparator.comparing;
+import static java.util.Comparator.reverseOrder;
 import static java.util.stream.Collectors.groupingBy;
 import static java.util.stream.Collectors.toList;
 import static java.util.stream.Collectors.toMap;
@@ -94,20 +99,57 @@ public class DeploymentTrigger {
         applications().lockApplicationIfPresent(id, application -> {
             DeploymentStatus status = jobs.deploymentStatus(application.get());
             for (InstanceName instanceName : application.get().deploymentSpec().instanceNames()) {
-                Change outstanding = status.outstandingChange(instanceName);
+                Change outstanding = outstandingChange(status, instanceName);
                 if (   outstanding.hasTargets()
                     && status.instanceSteps().get(instanceName)
                              .readyAt(outstanding)
                              .map(readyAt -> ! readyAt.isAfter(clock.instant())).orElse(false)
                     && acceptNewApplicationVersion(status, instanceName, outstanding.application().get())) {
                     application = application.with(instanceName,
-                                                   instance -> withRemainingChange(instance, instance.change().with(outstanding.application().get()), status));
+                                                   instance -> withRemainingChange(instance, outstanding.onTopOf(instance.change()), status));
                 }
             }
             applications().store(application);
         });
     }
 
+    /** Returns any outstanding change for the given instance, coupled with any necessary platform upgrade. */
+    private Change outstandingChange(DeploymentStatus status, InstanceName instance) {
+        Change outstanding = status.outstandingChange(instance);
+        Optional<Version> compileVersion = outstanding.application().flatMap(ApplicationVersion::compileVersion);
+
+        // If the outstanding revision requires a certain platform for compatibility, add that here.
+        VersionCompatibility compatibility = applications().versionCompatibility();
+        Predicate<Version> compatibleWithCompileVersion = version -> compileVersion.map(compiled -> compatibility.accept(version, compiled)).orElse(true);
+        if (status.application().productionDeployments().getOrDefault(instance, List.of()).stream()
+                  .anyMatch(deployment -> ! compatibleWithCompileVersion.test(deployment.version()))) {
+            return targetsForPolicy(controller.readVersionStatus(), status.application().deploymentSpec().requireInstance(instance).upgradePolicy())
+                    .stream() // Pick the latest platform which is compatible with the compile version, and is ready for this instance.
+                    .filter(compatibleWithCompileVersion)
+                    .map(outstanding::with)
+                    .filter(change -> status.instanceSteps().get(instance).readyAt(change)
+                                            .map(readyAt -> ! readyAt.isAfter(controller.clock().instant()))
+                                            .orElse(false))
+                    .findFirst().orElse(Change.empty());
+        }
+        return outstanding;
+    }
+
+    /** Returns target versions for given confidence, by descending version number. */
+    public List<Version> targetsForPolicy(VersionStatus versions, DeploymentSpec.UpgradePolicy policy) {
+        Version systemVersion = controller.systemVersion(versions);
+        if (policy == DeploymentSpec.UpgradePolicy.canary)
+            return List.of(systemVersion);
+
+        VespaVersion.Confidence target = policy == DeploymentSpec.UpgradePolicy.defaultPolicy ? VespaVersion.Confidence.normal : VespaVersion.Confidence.high;
+        return versions.versions().stream()
+                       .filter(version ->    ! version.versionNumber().isAfter(systemVersion)
+                                          &&   version.confidence().equalOrHigherThan(target))
+                       .map(VespaVersion::versionNumber)
+                       .sorted(reverseOrder())
+                       .collect(Collectors.toList());
+    }
+
     /**
      * Records information when a job completes (successfully or not). This information is used when deciding what to
      * trigger next.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 647b52f7d51..abd5ca8891a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -59,7 +59,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.Step.endStagingSetup;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.endTests;
 import static com.yahoo.vespa.hosted.controller.deployment.Step.report;
 import static java.time.temporal.ChronoUnit.SECONDS;
-import static java.util.Comparator.naturalOrder;
 import static java.util.function.Predicate.not;
 import static java.util.logging.Level.INFO;
 import static java.util.stream.Collectors.toList;
@@ -341,6 +340,7 @@ public class JobController {
                                                                    LinkedHashMap::new)),
                                     controller.system(),
                                     systemVersion,
+                                    controller.applications().versionCompatibility(),
                                     controller.clock().instant());
     }
 
@@ -496,6 +496,12 @@ public class JobController {
 
     /** Orders a run of the given type, or throws an IllegalStateException if that job type is already running. */
     public void start(ApplicationId id, JobType type, Versions versions, boolean isRedeployment, JobProfile profile, Optional<String> reason) {
+        if (versions.targetApplication().compileVersion()
+                    .map(version -> controller.applications().versionCompatibility().refuse(versions.targetPlatform(), version))
+                    .orElse(false))
+            throw new IllegalArgumentException("Will not start a job with incompatible platform version (" + versions.targetPlatform() + ") " +
+                                               "and compile versions (" + versions.targetApplication().compileVersion().get() + ")");
+
         locked(id, type, __ -> {
             Optional<Run> last = last(id, type);
             if (last.flatMap(run -> active(run.id())).isPresent())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index 8444ed8a374..f723d8b5134 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.component.Version;
+import com.yahoo.component.VersionCompatibility;
 import com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.curator.Lock;
@@ -91,10 +92,11 @@ public class Upgrader extends ControllerMaintainer {
                                                                                            : i -> i.failing()
                                                                                                    .not().upgradingTo(targetAndNewer);
 
+        VersionCompatibility compatibility = controller().applications().versionCompatibility();
         Map<ApplicationId, Version> targets = new LinkedHashMap<>();
-        for (Version version : targetsForPolicy(versionStatus, policy)) {
+        for (Version version : controller().applications().deploymentTrigger().targetsForPolicy(versionStatus, policy)) {
             targetAndNewer.add(version);
-            InstanceList eligible = eligibleForVersion(remaining, version, cancellationCriterion, targetMajorVersion);
+            InstanceList eligible = eligibleForVersion(remaining, version, targetMajorVersion, compatibility);
             InstanceList outdated = cancellationCriterion.apply(eligible);
             cancelUpgradesOf(outdated.upgrading(), "Upgrading to outdated versions");
 
@@ -111,25 +113,12 @@ public class Upgrader extends ControllerMaintainer {
         }
     }
 
-    /** Returns target versions for given confidence, by descending version number. */
-    private List<Version> targetsForPolicy(VersionStatus versions, UpgradePolicy policy) {
-        Version systemVersion = controller().systemVersion(versions);
-        if (policy == UpgradePolicy.canary)
-            return List.of(systemVersion);
-
-        Confidence target = policy == UpgradePolicy.defaultPolicy ? Confidence.normal : Confidence.high;
-        return versions.versions().stream()
-                .filter(version ->    ! version.versionNumber().isAfter(systemVersion)
-                                   &&   version.confidence().equalOrHigherThan(target))
-                .map(VespaVersion::versionNumber)
-                .sorted(reverseOrder())
-                .collect(Collectors.toList());
-    }
-
-    private InstanceList eligibleForVersion(InstanceList instances, Version version, UnaryOperator<InstanceList> cancellationCriterion, Optional<Integer> targetMajorVersion) {
+    private InstanceList eligibleForVersion(InstanceList instances, Version version,
+                                            Optional<Integer> targetMajorVersion, VersionCompatibility compatibility) {
         Change change = Change.of(version);
         return instances.not().failingOn(version)
-                        .allowMajorVersion(version.getMajor(), targetMajorVersion.orElse(version.getMajor()))
+                        .allowingMajorVersion(version.getMajor(), targetMajorVersion.orElse(version.getMajor()))
+                        .compatibleWithPlatform(version, compatibility)
                         .not().hasCompleted(change) // Avoid rescheduling change for instances without production steps.
                         .onLowerVersionThan(version)
                         .canUpgradeAt(version, controller().clock().instant());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index 5cf55930274..15a194d2ffa 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -17,6 +17,7 @@ import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzPrincipal;
 import com.yahoo.vespa.athenz.api.AthenzUser;
 import com.yahoo.vespa.athenz.api.OAuthCredentials;
+import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Property;
@@ -92,6 +93,7 @@ public final class ControllerTester {
     private final ServiceRegistryMock serviceRegistry;
     private final CuratorDb curator;
     private final RotationsConfig rotationsConfig;
+    private final InMemoryFlagSource flagSource;
     private final AtomicLong nextPropertyId = new AtomicLong(1000);
     private final AtomicInteger nextProjectId = new AtomicInteger(1000);
     private final AtomicInteger nextDomainId = new AtomicInteger(1000);
@@ -126,15 +128,17 @@ public final class ControllerTester {
         this(new AthenzDbMock(), new MockCuratorDb(), defaultRotationsConfig(), new ServiceRegistryMock(system));
     }
 
-    private ControllerTester(AthenzDbMock athenzDb, boolean inContainer,
-                             CuratorDb curator, RotationsConfig rotationsConfig,
-                             ServiceRegistryMock serviceRegistry, Controller controller) {
+    private ControllerTester(AthenzDbMock athenzDb, boolean inContainer, CuratorDb curator,
+                             RotationsConfig rotationsConfig, ServiceRegistryMock serviceRegistry,
+                             InMemoryFlagSource flagSource, Controller controller) {
         this.athenzDb = athenzDb;
         this.inContainer = inContainer;
         this.clock = serviceRegistry.clock();
         this.serviceRegistry = serviceRegistry;
         this.curator = curator;
         this.rotationsConfig = rotationsConfig;
+        this.flagSource = flagSource.withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true)
+                                    .withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of(), String.class);
         this.controller = controller;
 
         // Make root logger use time from manual clock
@@ -148,8 +152,14 @@ public final class ControllerTester {
     private ControllerTester(AthenzDbMock athenzDb,
                              CuratorDb curator, RotationsConfig rotationsConfig,
                              ServiceRegistryMock serviceRegistry) {
-        this(athenzDb, false, curator, rotationsConfig, serviceRegistry,
-             createController(curator, rotationsConfig, athenzDb, serviceRegistry));
+        this(athenzDb, curator, rotationsConfig, serviceRegistry, new InMemoryFlagSource());
+    }
+
+    private ControllerTester(AthenzDbMock athenzDb,
+                             CuratorDb curator, RotationsConfig rotationsConfig,
+                             ServiceRegistryMock serviceRegistry, InMemoryFlagSource flagSource) {
+        this(athenzDb, false, curator, rotationsConfig, serviceRegistry, flagSource,
+             createController(curator, rotationsConfig, athenzDb, serviceRegistry, flagSource));
     }
 
     /** Creates a ControllerTester built on the ContainerTester's controller. This controller can not be recreated. */
@@ -159,6 +169,7 @@ public final class ControllerTester {
              tester.controller().curator(),
              null,
              tester.serviceRegistry(),
+             tester.flagSource(),
              tester.controller());
     }
 
@@ -178,6 +189,8 @@ public final class ControllerTester {
 
     public AthenzDbMock athenzDb() { return athenzDb; }
 
+    public InMemoryFlagSource flagSource() { return flagSource; }
+
     public MemoryNameService nameService() {
         return serviceRegistry.nameService();
     }
@@ -217,10 +230,10 @@ public final class ControllerTester {
     }
 
     /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */
-    public final void createNewController() {
+    public void createNewController() {
         if (inContainer)
             throw new UnsupportedOperationException("Cannot recreate this controller");
-        controller = createController(curator, rotationsConfig, athenzDb, serviceRegistry);
+        controller = createController(curator, rotationsConfig, athenzDb, serviceRegistry, flagSource);
     }
 
     /** Upgrade controller to given version */
@@ -373,9 +386,8 @@ public final class ControllerTester {
 
     private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig,
                                                AthenzDbMock athensDb,
-                                               ServiceRegistryMock serviceRegistry) {
-        InMemoryFlagSource flagSource = new InMemoryFlagSource()
-                .withBooleanFlag(PermanentFlags.ENABLE_PUBLIC_SIGNUP_FLOW.id(), true);
+                                               ServiceRegistryMock serviceRegistry,
+                                               FlagSource flagSource) {
         Controller controller = new Controller(curator,
                                                rotationsConfig,
                                                serviceRegistry.zoneRegistry().system().isPublic() ?
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index c8126207a73..abdd4233339 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -2021,4 +2022,30 @@ public class DeploymentTriggerTest {
         List<RetriggerEntry> retriggerEntries = tester.controller().curator().readRetriggerEntries();
         Assert.assertEquals(1, retriggerEntries.size());
     }
+
+    @Test
+    public void testOrchestrationWithIncompatibleVersionPairs() {
+        Version version1 = new Version("7");
+        Version version2 = new Version("8");
+        tester.controllerTester().flagSource().withListFlag(PermanentFlags.INCOMPATIBLE_VERSIONS.id(), List.of("8"), String.class);
+
+        tester.controllerTester().upgradeSystem(version1);
+        DeploymentContext app = tester.newDeploymentContext()
+                                      .submit(new ApplicationPackageBuilder().region("us-east-3")
+                                                                             .compileVersion(version1)
+                                                                             .build())
+                                      .deploy();
+
+        tester.controllerTester().upgradeSystem(version2);
+        tester.upgrader().run();
+        assertEquals(Change.empty(), app.instance().change());
+
+        app.submit(new ApplicationPackageBuilder().region("us-east-3")
+                                                  .compileVersion(version2)
+                                                  .build());
+        app.deploy();
+        assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetPlatform());
+        assertEquals(version2, tester.jobs().last(app.instanceId(), productionUsEast3).get().versions().targetApplication().compileVersion().get());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
index 6165c2b8ca8..ca27e9d23cd 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
@@ -12,6 +12,7 @@ import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain;
 import com.yahoo.test.json.JsonTestHelper;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.ApplicationAction;
 import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock;
@@ -54,6 +55,10 @@ public class ContainerTester {
         return (AthenzClientFactoryMock) container.components().getComponent(AthenzClientFactoryMock.class.getName());
     }
 
+    public InMemoryFlagSource flagSource() {
+        return (InMemoryFlagSource) container.components().getComponent(InMemoryFlagSource.class.getName());
+    }
+
     public ServiceRegistryMock serviceRegistry() {
         return (ServiceRegistryMock) container.components().getComponent(ServiceRegistryMock.class.getName());
     }
@@ -110,8 +115,8 @@ public class ContainerTester {
                     .replaceAll("\"?\\(ignore\\)\"?", "\\\\E" +
                             stopCharacters + "*\\\\Q");
             if (!Pattern.matches(expectedResponsePattern, responseString)) {
-                throw new ComparisonFailure(responseFile.toString() + " (with ignored fields)",
-                        expectedResponsePattern, responseString);
+                throw new ComparisonFailure(responseFile + " (with ignored fields)",
+                                            expectedResponsePattern, responseString);
             }
         } else {
             if (compareJson) {
@@ -210,5 +215,4 @@ public class ContainerTester {
     public interface ConsumerThrowingException<T> {
         void accept(T t) throws Exception;
     }
-}
-    
+}
\ No newline at end of file
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/InMemoryFlagSource.java b/flags/src/main/java/com/yahoo/vespa/flags/InMemoryFlagSource.java
index 9a69adfc1a6..d21e21b44c8 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/InMemoryFlagSource.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/InMemoryFlagSource.java
@@ -1,6 +1,8 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.flags;
 
+import com.yahoo.component.AbstractComponent;
+
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -14,7 +16,7 @@ import java.util.concurrent.ConcurrentHashMap;
  *
  * @author freva
  */
-public class InMemoryFlagSource implements FlagSource {
+public class InMemoryFlagSource extends AbstractComponent implements FlagSource {
     private final Map<FlagId, RawFlag> rawFlagsById = new ConcurrentHashMap<>();
 
     public InMemoryFlagSource withBooleanFlag(FlagId flagId, boolean value) {
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
index 225c45fbfdb..72a16a37a8f 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/PermanentFlags.java
@@ -240,6 +240,20 @@ public class PermanentFlags {
             "Takes effect immediately.",
             ZONE_ID, APPLICATION_ID);
 
+    public static final UnboundListFlag<String> INCOMPATIBLE_VERSIONS = defineListFlag(
+            "incompatible-versions", List.of("8"), String.class,
+            "A list of versions which are binary-incompatible with earlier versions. " +
+            "A platform version A and an application package compiled against version B are thus incompatible if this " +
+            "list contains a version X such that (A >= X) != (B >= X). " +
+            "A version specifying only major, or major and minor, imply 0s for the unspecified parts." +
+            "This list may also contain '*' wildcards for any suffix of its version number; see the VersionCompatibility " +
+            "class for further details. " +
+            "The controller will attempt to couple platform upgrades to application changes if their compile versions are " +
+            "incompatible with any current deployments. " +
+            "The config server will refuse to serve config to nodes running a version which is incompatible with their " +
+            "current wanted node version, i.e., nodes about to upgrade to a version which is incompatible with the current.",
+            "Takes effect immediately");
+
     public static final UnboundListFlag<Integer> INCOMPATIBLE_MAJOR_VERSIONS = defineListFlag(
             "incompatible-major-versions", List.of(8), Integer.class,
             "A list of major versions which are binary-incompatible and requires an application package to " +