Merge pull request #8058 from vespa-engine/jvenstad/pin-versions

Jvenstad/pin versions

15 files changed, 202 insertions, 63 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index ab97db06484..3185b7d5f45 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -671,7 +671,7 @@ public class ApplicationController {
     private void validateRun(Application application, ZoneId zone, Version platformVersion, ApplicationVersion applicationVersion) {
         Deployment deployment = application.deployments().get(zone);
         if (   zone.environment().isProduction() && deployment != null
-            && (   platformVersion.compareTo(deployment.version()) < 0
+            && (   platformVersion.compareTo(deployment.version()) < 0 && ! application.change().isPinned()
                 || applicationVersion.compareTo(deployment.applicationVersion()) < 0))
             throw new IllegalArgumentException(String.format("Rejecting deployment of %s to %s, as the requested versions (platform: %s, application: %s)" +
                                                              " are older than the currently deployed (platform: %s, application: %s).",
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index 9e8b2495616..677b4b4ba07 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -63,6 +63,11 @@ public class ApplicationList {
         return listOf(list.stream().filter(application -> isUpgradingTo(version, application)));
     }
 
+    /** Returns the subset of applications which are not pinned to a certain Vespa version. */
+    public ApplicationList notPinned() {
+        return listOf(list.stream().filter(application -> ! application.change().isPinned()));
+    }
+
     /** Returns the subset of applications which are currently not upgrading to the given version */
     public ApplicationList notUpgradingTo(Version version) {
         return listOf(list.stream().filter(application -> ! isUpgradingTo(version, application)));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
index ab16f84f628..7b77105cc67 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Change.java
@@ -20,7 +20,7 @@ import java.util.StringJoiner;
  */
 public final class Change {
 
-    private static final Change empty = new Change(Optional.empty(), Optional.empty());
+    private static final Change empty = new Change(Optional.empty(), Optional.empty(), false);
 
     /** The platform version we are upgrading to, or empty if none */
     private final Optional<Version> platform;
@@ -28,7 +28,9 @@ public final class Change {
     /** The application version we are changing to, or empty if none */
     private final Optional<ApplicationVersion> application;
 
-    private Change(Optional<Version> platform, Optional<ApplicationVersion> application) {
+    private final boolean pinned;
+
+    private Change(Optional<Version> platform, Optional<ApplicationVersion> application, boolean pinned) {
         Objects.requireNonNull(platform, "platform cannot be null");
         Objects.requireNonNull(application, "application cannot be null");
         if (application.isPresent() && application.get().isUnknown()) {
@@ -36,14 +38,15 @@ public final class Change {
         }
         this.platform = platform;
         this.application = application;
+        this.pinned = pinned;
     }
 
     public Change withoutPlatform() {
-        return new Change(Optional.empty(), application);
+        return new Change(Optional.empty(), application, pinned);
     }
 
     public Change withoutApplication() {
-        return new Change(platform, Optional.empty());
+        return new Change(platform, Optional.empty(), pinned);
     }
 
     /** Returns whether a change should currently be deployed */
@@ -57,17 +60,32 @@ public final class Change {
     /** Returns the application version carried by this. */
     public Optional<ApplicationVersion> application() { return application; }
 
+    public boolean isPinned() { return pinned; }
+
     /** Returns an instance representing no change */
     public static Change empty() { return empty; }
 
     /** Returns a version of this change which replaces or adds this platform change */
     public Change with(Version platformVersion) {
-        return new Change(Optional.of(platformVersion), application);
+        if (pinned)
+            throw new IllegalArgumentException("Not allowed to set a platform version when pinned.");
+
+        return new Change(Optional.of(platformVersion), application, pinned);
     }
 
     /** Returns a version of this change which replaces or adds this application change */
     public Change with(ApplicationVersion applicationVersion) {
-        return new Change(platform, Optional.of(applicationVersion));
+        return new Change(platform, Optional.of(applicationVersion), pinned);
+    }
+
+    /** Returns a change with the versions of this, and with the platform version pinned. */
+    public Change withPin() {
+        return new Change(platform, application, true);
+    }
+
+    /** Returns a change with the versions of this, and with the platform version unpinned. */
+    public Change withoutPin() {
+        return new Change(platform, application, false);
     }
 
     /** Returns the change obtained when overwriting elements of the given change with any present in this */
@@ -76,37 +94,44 @@ public final class Change {
             other = other.with(platform.get());
         if (application.isPresent())
             other = other.with(application.get());
+        if (pinned)
+            other = other.withPin();
         return other;
     }
 
     @Override
-    public int hashCode() { return Objects.hash(platform, application); }
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (!(o instanceof Change)) return false;
+        Change change = (Change) o;
+        return pinned == change.pinned &&
+               Objects.equals(platform, change.platform) &&
+               Objects.equals(application, change.application);
+    }
 
     @Override
-    public boolean equals(Object other) {
-        if (other == this) return true;
-        if ( ! (other instanceof Change)) return false;
-        Change o = (Change)other;
-        if ( ! o.platform.equals(this.platform)) return false;
-        if ( ! o.application.equals(this.application)) return false;
-        return true;
+    public int hashCode() {
+        return Objects.hash(platform, application, pinned);
     }
 
     @Override
     public String toString() {
         StringJoiner changes = new StringJoiner(" and ");
-        platform.ifPresent(version -> changes.add("upgrade to " + version.toString()));
+        if (pinned)
+            changes.add("pin to " + platform.map(Version::toString).orElse("current platform"));
+        else
+            platform.ifPresent(version -> changes.add("upgrade to " + version.toString()));
         application.ifPresent(version -> changes.add("application change to " + version.id()));
         changes.setEmptyValue("no change");
         return changes.toString();
     }
 
     public static Change of(ApplicationVersion applicationVersion) {
-        return new Change(Optional.empty(), Optional.of(applicationVersion));
+        return new Change(Optional.empty(), Optional.of(applicationVersion), false);
     }
 
     public static Change of(Version platformChange) {
-        return new Change(Optional.of(platformChange), Optional.empty());
+        return new Change(Optional.of(platformChange), Optional.empty(), false);
     }
 
     /** Returns whether this change carries an application downgrade relative to the given version. */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 3c51640cb01..f6f843cabcb 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -256,15 +256,17 @@ public class DeploymentTrigger {
             Change change;
             switch (cancellation) {
                 case ALL: change = Change.empty(); break;
+                case VERSIONS: change = Change.empty().withPin(); break;
                 case PLATFORM: change = application.get().change().withoutPlatform(); break;
                 case APPLICATION: change = application.get().change().withoutApplication(); break;
+                case PIN: change = application.get().change().withoutPin(); break;
                 default: throw new IllegalArgumentException("Unknown cancellation choice '" + cancellation + "'!");
             }
             applications().store(application.withChange(change));
         });
     }
 
-    public enum ChangesToCancel { ALL, PLATFORM, APPLICATION }
+    public enum ChangesToCancel { ALL, PLATFORM, APPLICATION, VERSIONS, PIN }
 
     // ---------- Conveniences ----------
 
@@ -456,9 +458,17 @@ public class DeploymentTrigger {
      * is already deployed in its zone, i.e., no parts of the change are upgrades, and the full current
      * change for the application downgrades the deployment, which is an acknowledgement that the deployed
      * version is broken somehow, such that the job may be locked in failure until a new version is released.
+     *
+     * Additionally, if the application is pinned to a Vespa version, and the given change has a (this) platform,
+     * the deployment for the job must be on the pinned version.
      */
     public boolean isComplete(Change change, Application application, JobType jobType) {
         Optional<Deployment> existingDeployment = deploymentFor(application, jobType);
+        if (     change.isPinned()
+            &&   change.platform().isPresent()
+            && ! existingDeployment.map(Deployment::version).equals(change.platform()))
+            return false;
+
         return       application.deploymentJobs().statusOf(jobType).flatMap(JobStatus::lastSuccess)
                                 .map(job ->    change.platform().map(job.platform()::equals).orElse(true)
                                             && change.application().map(job.application()::equals).orElse(true))
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
index d0cc6ee5a6f..bef61dda875 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Versions.java
@@ -127,6 +127,9 @@ public class Versions {
 
     private static Version targetPlatform(Application application, Change change, Optional<Deployment> deployment,
                                           Version defaultVersion) {
+        if (change.isPinned() && change.platform().isPresent())
+            return change.platform().get();
+
         return max(change.platform(), deployment.map(Deployment::version))
                 .orElse(application.oldestDeployedPlatform()
                                    .orElse(defaultVersion));
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index c1665dfba42..3aa61fe8370 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -92,8 +92,10 @@ public class Upgrader extends Maintainer {
         return reversed;
     }
 
-    /** Returns a list of all applications */
-    private ApplicationList applications() { return ApplicationList.from(controller().applications().asList()); }
+    /** Returns a list of all applications, except those which are pinned — these should not be manipulated by the Upgrader */
+    private ApplicationList applications() {
+        return ApplicationList.from(controller().applications().asList()).notPinned();
+    }
 
     private void upgrade(ApplicationList applications, Version version) {
         applications = applications.hasProductionDeployment();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
index f4d87d7c967..69ea36c7e3a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java
@@ -60,6 +60,7 @@ public class ApplicationSerializer {
     private final String deploymentsField = "deployments";
     private final String deploymentJobsField = "deploymentJobs";
     private final String deployingField = "deployingField";
+    private final String pinnedField = "pinned";
     private final String outstandingChangeField = "outstandingChangeField";
     private final String ownershipIssueIdField = "ownershipIssueId";
     private final String ownerField = "confirmedOwner";
@@ -275,13 +276,15 @@ public class ApplicationSerializer {
     }
 
     private void toSlime(Change deploying, Cursor parentObject, String fieldName) {
-        if ( ! deploying.isPresent()) return;
+        if ( ! deploying.isPresent() && ! deploying.isPinned()) return;
 
         Cursor object = parentObject.setObject(fieldName);
         if (deploying.platform().isPresent())
             object.setString(versionField, deploying.platform().get().toString());
         if (deploying.application().isPresent())
             toSlime(deploying.application().get(), object);
+        if (deploying.isPinned())
+            object.setBool(pinnedField, true);
     }
 
     private void toSlime(Map<HostName, RotationStatus> rotationStatus, Cursor array) {
@@ -435,8 +438,8 @@ public class ApplicationSerializer {
             change = Change.of(Version.fromString(versionFieldValue.asString()));
         if (object.field(applicationBuildNumberField).valid())
             change = change.with(applicationVersionFromSlime(object));
-        if ( ! change.isPresent()) // A deploy object with no fields -> unknown application change
-            change = Change.empty();
+        if (object.field(pinnedField).asBool())
+            change = change.withPin();
         return change;
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index afaa0e88d0e..c8104c6e2b0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -68,6 +68,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.RotationStatus;
 import com.yahoo.vespa.hosted.controller.athenz.impl.ZmsClientFacade;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
@@ -100,7 +101,6 @@ import java.util.Optional;
 import java.util.Scanner;
 import java.util.logging.Level;
 
-import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL;
 import static java.util.stream.Collectors.joining;
 
 /**
@@ -203,7 +203,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request);
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploy(path.get("tenant"), path.get("application"), request);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), readToString(request.getData()), false);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), readToString(request.getData()), true);
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request);
@@ -219,7 +221,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request);
-        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"));
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all");
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("choice"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -755,48 +758,63 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     /**
      *  Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9",
-     *  or the latest known commit of the application if "commit" is given,
-     *  or an upgrade to the system version if no data is provided.
+     *  optionally pinning to that version if.
      */
-    private HttpResponse deploy(String tenantName, String applicationName, HttpRequest request) {
+    private HttpResponse deployPlatform(String tenantName, String applicationName, String versionString, boolean pin) {
         ApplicationId id = ApplicationId.from(tenantName, applicationName, "default");
-        String requestVersion = readToString(request.getData());
         StringBuilder response = new StringBuilder();
         controller.applications().lockOrThrow(id, application -> {
-            Change change;
-            if ("commit".equals(requestVersion))
-                change = Change.of(application.get().deploymentJobs().statusOf(JobType.component)
-                                              .get().lastSuccess().get().application());
-            else {
-                Version version = requestVersion == null ? controller.systemVersion() : new Version(requestVersion);
-                if ( ! systemHasVersion(version))
-                    throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " +
-                                                               "Version is not active in this system. " +
-                                                               "Active versions: " + controller.versionStatus().versions()
-                                                                                               .stream()
-                                                                                               .map(VespaVersion::versionNumber)
-                                                                                               .map(Version::toString)
-                                                                                               .collect(joining(", ")));
-                change = Change.of(version);
-            }
+            Version version = Version.fromString(versionString);
+            if (version.equals(Version.emptyVersion))
+                version = controller.systemVersion();
+            if ( ! systemHasVersion(version))
+                throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " +
+                                                       "Version is not active in this system. " +
+                                                       "Active versions: " + controller.versionStatus().versions()
+                                                                                       .stream()
+                                                                                       .map(VespaVersion::versionNumber)
+                                                                                       .map(Version::toString)
+                                                                                       .collect(joining(", ")));
+            Change change = Change.of(version);
+            if (pin)
+                change = change.withPin();
+
+            controller.applications().deploymentTrigger().forceChange(id, change);
+            response.append("Triggered " + change + " for " + id);
+        });
+        return new MessageResponse(response.toString());
+    }
+
+    /** Trigger deployment to the last known application package for the given application. */
+    private HttpResponse deployApplication(String tenantName, String applicationName) {
+        ApplicationId id = ApplicationId.from(tenantName, applicationName, "default");
+        StringBuilder response = new StringBuilder();
+        controller.applications().lockOrThrow(id, application -> {
+            Change change = Change.of(application.get().deploymentJobs().statusOf(JobType.component).get().lastSuccess().get().application());
             controller.applications().deploymentTrigger().forceChange(id, change);
             response.append("Triggered " + change + " for " + id);
         });
         return new MessageResponse(response.toString());
     }
 
-    /** Cancel any ongoing change for given application */
-    private HttpResponse cancelDeploy(String tenantName, String applicationName) {
+    /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */
+    private HttpResponse cancelDeploy(String tenantName, String applicationName, String choice) {
         ApplicationId id = ApplicationId.from(tenantName, applicationName, "default");
-        Application application = controller.applications().require(id);
-        Change change = application.change();
-        if ( ! change.isPresent())
-            return new MessageResponse("No deployment in progress for " + application + " at this time");
+        StringBuilder response = new StringBuilder();
+        controller.applications().lockOrThrow(id, application -> {
+            Change change = application.get().change();
+            if ( ! change.isPresent()) {
+                response.append("No deployment in progress for " + application + " at this time");
+                return;
+            }
 
-        controller.applications().lockOrThrow(id, lockedApplication ->
-                controller.applications().deploymentTrigger().cancelChange(id, ALL));
+            ChangesToCancel cancel = ChangesToCancel.valueOf(choice.toUpperCase());
+            controller.applications().deploymentTrigger().cancelChange(id, cancel);
+            response.append("Changed deployment from '" + change + "' to '" +
+                            controller.applications().require(id).change() + "' for " + application);
+        });
 
-        return new MessageResponse("Cancelled " + change + " for " + application);
+        return new MessageResponse(response.toString());
     }
 
     /** Schedule restart of deployment, or specific host in a deployment */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java
index bc628818fbe..b8e868ed8a4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java
@@ -131,7 +131,7 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase {
         if (isHostedOperatorOperation(path, method)) return false;
         return path.matches("/application/v4/tenant/{tenant}") ||
                path.matches("/application/v4/tenant/{tenant}/application/{application}") ||
-               path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") ||
+               path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{*}") ||
                path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{job}/{*}") ||
                path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/dev/{*}") ||
                path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/perf/{*}") ||
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index b81d9e78339..6755a6f9ad5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -10,8 +10,10 @@ import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import org.junit.Before;
@@ -29,6 +31,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
 import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.ALL;
+import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PIN;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
@@ -1132,4 +1135,67 @@ public class UpgraderTest {
         assertFalse(tester.application(app.id()).change().isPresent());
     }
 
+    @Test
+    public void testPinning() {
+        Version version0 = Version.fromString("6.2");
+        tester.upgradeSystem(version0);
+
+        // Create an application with pinned platform version.
+        Application application = tester.createApplication("application", "tenant", 2, 3);
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod)
+                                                                               .region("us-east-3")
+                                                                               .region("us-west-1")
+                                                                               .build();
+        tester.deploymentTrigger().forceChange(application.id(), Change.empty().withPin());
+
+        tester.deployCompletely(application, applicationPackage);
+        assertFalse(tester.application(application.id()).change().isPresent());
+        assertTrue(tester.application(application.id()).change().isPinned());
+        assertEquals(2, tester.application(application.id()).deployments().size());
+
+        // Application does not upgrade.
+        Version version1 = Version.fromString("6.3");
+        tester.upgradeSystem(version1);
+        tester.upgrader().maintain();
+        assertFalse(tester.application(application.id()).change().isPresent());
+        assertTrue(tester.application(application.id()).change().isPinned());
+
+        // New application package is deployed.
+        tester.deployCompletely(application, applicationPackage, BuildJob.defaultBuildNumber + 1);
+        assertFalse(tester.application(application.id()).change().isPresent());
+        assertTrue(tester.application(application.id()).change().isPinned());
+
+        // Application upgrades to new version when pin is removed.
+        tester.deploymentTrigger().cancelChange(application.id(), PIN);
+        tester.upgrader().maintain();
+        assertTrue(tester.application(application.id()).change().isPresent());
+        assertFalse(tester.application(application.id()).change().isPinned());
+
+        // Application is pinned to new version, and upgrade is therefore not cancelled, even though confidence is broken.
+        tester.deploymentTrigger().forceChange(application.id(), Change.empty().withPin());
+        tester.upgrader().maintain();
+        tester.readyJobTrigger().maintain();
+        assertEquals(version1, tester.application(application.id()).change().platform().get());
+
+        // Application fails upgrade after one zone is complete, and is pinned again to the old version.
+        tester.deployAndNotify(application, true, systemTest);
+        tester.deployAndNotify(application, true, stagingTest);
+        tester.deployAndNotify(application, true, productionUsEast3);
+        tester.deploy(productionUsWest1, application, Optional.empty(), false);
+        tester.deployAndNotify(application, false, productionUsWest1);
+        tester.deploymentTrigger().cancelChange(application.id(), ALL);
+        tester.deploymentTrigger().forceChange(application.id(), Change.of(version0).withPin());
+        tester.buildService().clear();
+        assertEquals(version0, tester.application(application.id()).change().platform().get());
+
+        // Application downgrades to pinned version.
+        tester.readyJobTrigger().maintain();
+        tester.deployAndNotify(application, true, systemTest);
+        tester.deployAndNotify(application, true, stagingTest);
+        tester.deployAndNotify(application, true, productionUsEast3);
+        assertTrue(tester.application(application.id()).change().isPresent());
+        tester.deployAndNotify(application, true, productionUsWest1);
+        assertFalse(tester.application(application.id()).change().isPresent());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
index 7dff2667b9f..774caea97b0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java
@@ -104,7 +104,7 @@ public class ApplicationSerializerTest {
                                                deploymentSpec,
                                                validationOverrides,
                                                deployments, deploymentJobs,
-                                               Change.of(Version.fromString("6.7")),
+                                               Change.of(Version.fromString("6.7")).withPin(),
                                                Change.of(ApplicationVersion.from(new SourceRevision("repo", "master", "deadcafe"), 42)),
                                                Optional.of(IssueId.from("1234")),
                                                Optional.of(User.from("by-username")),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index d9950553e38..d455218f4e9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -365,15 +365,21 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
         // DELETE (cancel) again is a no-op
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE)
-                                      .userIdentity(USER_ID),
+                                      .userIdentity(USER_ID)
+                                      .data("{\"cancel\":\"all\"}"),
                               new File("application-deployment-cancelled-no-op.json"));
 
-        // POST triggering of a full deployment to an application (if version is omitted, current system version is used)
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", POST)
+        // POST pinning to a given version to an application
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST)
                                       .userIdentity(USER_ID)
                                       .data("6.1.0"),
                               new File("application-deployment.json"));
 
+        // DELETE only the pin to a given version
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE)
+                                      .userIdentity(USER_ID),
+                              new File("application-pin-cancelled.json"));
+
         // POST a pause to a production job
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST)
                                       .userIdentity(USER_ID),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json
index bc09003d86f..efca5831256 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment-cancelled.json
@@ -1 +1 @@
-{"message":"Cancelled application change to 1.0.42-commit1 for application 'tenant1.application1'"}
+{"message":"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'"}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json
index d2531638a93..fe68f3d94a3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-deployment.json
@@ -1 +1 @@
-{"message":"Triggered upgrade to 6.1 for tenant1.application1"}
\ No newline at end of file
+{"message":"Triggered pin to 6.1 for tenant1.application1"}
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json
new file mode 100644
index 00000000000..62360458ce4
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-pin-cancelled.json
@@ -0,0 +1 @@
+{"message":"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'"}