Merge master to get good unit tests

20 files changed, 682 insertions, 100 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index ab1b5aa3cd4..a04eee573c0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -116,6 +116,12 @@ public class ApplicationList {
         return listOf(list.stream().filter(a -> a.deploymentSpec().canUpgradeAt(instant)));
     }
 
+    /** Returns the first n application in this (or all, if there are less than n). */
+    public ApplicationList first(int n) {
+        if (list.size() < n) return this;
+        return new ApplicationList(list.subList(0, n));
+    }
+
      // ----------------------------------- Sorting
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
index 82e6ca919e1..48ebbc9f972 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentOrder.java
@@ -1,9 +1,13 @@
 package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.Zone;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobType;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 
@@ -11,10 +15,15 @@ import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
 import java.util.Collections;
+import java.util.Comparator;
+import java.util.LinkedHashMap;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.function.Function;
 import java.util.logging.Logger;
+import java.util.stream.Collector;
 import java.util.stream.Collectors;
 
 import static java.util.stream.Collectors.collectingAndThen;
@@ -104,6 +113,27 @@ public class DeploymentOrder {
                 .collect(Collectors.collectingAndThen(Collectors.toList(), Collections::unmodifiableList));
     }
 
+    /** Returns job status sorted according to deployment spec */
+    public Map<JobType, JobStatus> sortBy(DeploymentSpec deploymentSpec, Map<JobType, JobStatus> jobStatus) {
+        List<DeploymentJobs.JobType> jobs = jobsFrom(deploymentSpec);
+        return jobStatus.entrySet().stream()
+                .sorted(Comparator.comparingInt(kv -> jobs.indexOf(kv.getKey())))
+                .collect(Collectors.collectingAndThen(toLinkedMap(Map.Entry::getKey, Map.Entry::getValue),
+                                                      Collections::unmodifiableMap));
+    }
+
+    /** Returns deployments sorted according to declared zones */
+    public Map<Zone, Deployment> sortBy(List<DeploymentSpec.DeclaredZone> zones, Map<Zone, Deployment> deployments) {
+        List<Zone> productionZones = zones.stream()
+                .filter(z -> z.environment() == Environment.prod && z.region().isPresent())
+                .map(z -> new Zone(z.environment(), z.region().get()))
+                .collect(Collectors.toList());
+        return deployments.entrySet().stream()
+                .sorted(Comparator.comparingInt(kv -> productionZones.indexOf(kv.getKey())))
+                .collect(Collectors.collectingAndThen(toLinkedMap(Map.Entry::getKey, Map.Entry::getValue),
+                                                      Collections::unmodifiableMap));
+    }
+
     /** Returns jobs for the given step */
     private List<JobType> jobsFrom(DeploymentSpec.Step step) {
         return step.zones().stream()
@@ -166,4 +196,13 @@ public class DeploymentOrder {
         return totalDelay;
     }
 
+    private static <T, K, U> Collector<T, ?, Map<K,U>> toLinkedMap(Function<? super T, ? extends K> keyMapper,
+                                                                   Function<? super T, ? extends U> valueMapper) {
+        return Collectors.toMap(keyMapper, valueMapper,
+                                (u, v) -> {
+                                    throw new IllegalStateException(String.format("Duplicate key %s", u));
+                                },
+                                LinkedHashMap::new);
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
index 268965850ba..423d8c674df 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java
@@ -102,9 +102,8 @@ public class DeploymentTrigger {
     public void triggerFailing(ApplicationId applicationId, Duration timeout) {
         try (Lock lock = applications().lock(applicationId)) {
             Application application = applications().require(applicationId);
-            if (!application.deploying().isPresent()) { // No ongoing change, no need to retry
-                return;
-            }
+            if (!application.deploying().isPresent()) return; // No ongoing change, no need to retry
+
             // Retry first failing job
             for (JobType jobType : order.jobsFrom(application.deploymentSpec())) {
                 JobStatus jobStatus = application.deploymentJobs().jobStatus().get(jobType);
@@ -116,6 +115,7 @@ public class DeploymentTrigger {
                     break;
                 }
             }
+
             // Retry dead job
             Optional<JobStatus> firstDeadJob = firstDeadJob(application.deploymentJobs(), timeout);
             if (firstDeadJob.isPresent()) {
@@ -214,16 +214,15 @@ public class DeploymentTrigger {
     /** Decide whether the job should be triggered by the periodic trigger */
     private boolean shouldRetryNow(JobStatus job) {
         if (job.isSuccess()) return false;
+        if (job.inProgress()) return false;
 
-        if ( ! job.lastCompleted().isPresent()) return true; // Retry when we don't hear back
+        // Retry after 10% of the time since it started failing
+        Duration aTenthOfFailTime = Duration.ofMillis( (clock.millis() - job.firstFailing().get().at().toEpochMilli()) / 10);
+        if (job.lastCompleted().get().at().isBefore(clock.instant().minus(aTenthOfFailTime))) return true;
 
-        // Always retry if we haven't tried in 4 hours
+        // ... or retry anyway if we haven't tried in 4 hours
         if (job.lastCompleted().get().at().isBefore(clock.instant().minus(Duration.ofHours(4)))) return true;
 
-        // Wait for 10% of the time since it started failing
-        Duration aTenthOfFailTime = Duration.ofMillis( (clock.millis() - job.firstFailing().get().at().toEpochMilli()) / 10);
-        if (job.lastCompleted().get().at().isBefore(clock.instant().minus(aTenthOfFailTime))) return true;
-        
         return false;
     }
     
@@ -309,4 +308,6 @@ public class DeploymentTrigger {
 
     public BuildSystem buildSystem() { return buildSystem; }
 
+    public DeploymentOrder deploymentOrder() { return order; }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java
index 0d9330ed8ea..bb3c4314e0d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystem.java
@@ -26,10 +26,12 @@ public class PolledBuildSystem implements BuildSystem {
 
     private static final Logger log = Logger.getLogger(PolledBuildSystem.class.getName());
 
-    private final Controller controller;
+    // The number of jobs to offer, on each poll, for zones that have limited capacity
+    private static final int maxCapacityConstraintedJobsToOffer = 2;
 
+    private final Controller controller;
     private final CuratorDb curator;
-    
+
     public PolledBuildSystem(Controller controller, CuratorDb curator) {
         this.controller = controller;
         this.curator = curator;
@@ -75,6 +77,7 @@ public class PolledBuildSystem implements BuildSystem {
     }
     
     private List<BuildJob> getJobs(boolean removeFromQueue) {
+        int capacityConstrainedJobsOffered = 0;
         try (Lock lock = curator.lockJobQueues()) {
             List<BuildJob> jobsToRun = new ArrayList<>();
             for (JobType jobType : JobType.values()) {
@@ -90,8 +93,11 @@ public class PolledBuildSystem implements BuildSystem {
                                     " because project ID is missing");
                     }
 
-                    // Return only one job at a time for capacity constrained queues
-                    if (removeFromQueue && isCapacityConstrained(jobType)) break;
+                    // Return a limited number of jobs at a time for capacity constrained zones
+                    if (removeFromQueue && isCapacityConstrained(jobType) &&
+                        ++capacityConstrainedJobsOffered >= maxCapacityConstraintedJobsToOffer) {
+                        break;
+                    }
                 }
                 if (removeFromQueue)
                     curator.writeJobQueue(jobType, queue);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 016ea66cb1a..1b692ecf243 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.Issues;
 import com.yahoo.vespa.hosted.controller.api.integration.Properties;
 import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.time.Duration;
 
@@ -33,7 +34,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final DelayedDeployer delayedDeployer;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
-    public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller,
+    public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator,
                                  JobControl jobControl, Metric metric, Chef chefClient,
                                  Contacts contactsClient, Properties propertiesClient, Issues issuesClient) {
         Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
@@ -45,9 +46,11 @@ public class ControllerMaintenance extends AbstractComponent {
         failureRedeployer = new FailureRedeployer(controller, maintenanceInterval, jobControl);
         outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl);
         versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl);
-        upgrader = new Upgrader(controller, maintenanceInterval, jobControl);
+        upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
         delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl);
     }
+
+    public Upgrader upgrader() { return upgrader; }
     
     /** Returns control of the maintenance jobs of this */
     public JobControl jobControl() { return jobControl; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index cd1d724de9d..9b4ca083df2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
 
@@ -27,8 +28,11 @@ public class Upgrader extends Maintainer {
 
     private static final Logger log = Logger.getLogger(Upgrader.class.getName());
 
-    public Upgrader(Controller controller, Duration interval, JobControl jobControl) {
+    private final CuratorDb curator;
+
+    public Upgrader(Controller controller, Duration interval, JobControl jobControl, CuratorDb curator) {
         super(controller, interval, jobControl);
+        this.curator = curator;
     }
 
     /**
@@ -38,7 +42,7 @@ public class Upgrader extends Maintainer {
     public void maintain() {
         VespaVersion target = controller().versionStatus().version(controller().systemVersion());
         if (target == null) return; // we don't have information about the current system version at this time
-        
+
         switch (target.confidence()) {
             case broken:
                 ApplicationList toCancel = applications().upgradingTo(target.versionNumber())
@@ -74,7 +78,9 @@ public class Upgrader extends Maintainer {
         applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version
         applications = applications.notCurrentlyUpgrading(change, startOfUpgradePeriod); // do not trigger again if currently upgrading
         applications = applications.canUpgradeAt(controller().clock().instant()); // wait with applications that are currently blocking upgrades
-        for (Application application : applications.byIncreasingDeployedVersion().asList()) {
+        applications = applications.byIncreasingDeployedVersion(); // start with lowest versions
+        applications = applications.first(numberOfApplicationsToUpgrade()); // throttle upgrades
+        for (Application application : applications.asList()) {
             try {
                 controller().applications().deploymentTrigger().triggerChange(application.id(), change);
             } catch (IllegalArgumentException e) {
@@ -89,4 +95,19 @@ public class Upgrader extends Maintainer {
         }
     }
 
+    /** Returns the number of applications to upgrade in this run */
+    private int numberOfApplicationsToUpgrade() {
+        return Math.max(1, (int)(maintenanceInterval().getSeconds() * (upgradesPerMinute() / 60)));
+    }
+
+    /** Returns number upgrades per minute */
+    public double upgradesPerMinute() {
+        return curator.readUpgradesPerMinute();
+    }
+
+    /** Sets the number upgrades per minute */
+    public void setUpgradesPerMinute(double n) {
+        curator.writeUpgradesPerMinute(n);
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index a83a24764ce..66821b22458 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.zookeeper.ZooKeeperServer;
 
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayDeque;
@@ -192,6 +193,23 @@ public class CuratorDb {
         transaction.commit();
     }
 
+    public double readUpgradesPerMinute() {
+        Optional<byte[]> n = curator.getData(upgradePerMinutePath());
+        if (!n.isPresent() || n.get().length == 0) {
+            return 0.5; // Default if value has never been written
+        }
+        return ByteBuffer.wrap(n.get()).getDouble();
+    }
+
+    public void writeUpgradesPerMinute(double n) {
+        if (n < 0) {
+            throw new IllegalArgumentException("Upgrades per minute must be >= 0");
+        }
+        NestedTransaction transaction = new NestedTransaction();
+        curator.set(upgradePerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array());
+        transaction.commit();
+    }
+
     // -------------- Paths --------------------------------------------------
 
     private Path systemVersionPath() {
@@ -222,4 +240,8 @@ public class CuratorDb {
         return root.append("jobQueues").append(jobType.name());
     }
 
+    private Path upgradePerMinutePath() {
+        return root.append("upgrader").append("upgradesPerMinute");
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 7aef1e413aa..c2777a78f32 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -62,6 +62,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.ApplicationRevision;
 import com.yahoo.vespa.hosted.controller.application.Change;
 import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.application.JobStatus;
 import com.yahoo.vespa.hosted.controller.application.SourceRevision;
 import com.yahoo.vespa.hosted.controller.common.NotFoundCheckedException;
@@ -97,7 +98,9 @@ import java.util.logging.Level;
  * on hosted Vespa.
  * 
  * @author bratseth
+ * @author mpolden
  */
+@SuppressWarnings("unused") // created by injection
 public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private final Controller controller;
@@ -122,7 +125,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 case PUT: return handlePUT(request);
                 case POST: return handlePOST(request);
                 case DELETE: return handleDELETE(request);
-                case OPTIONS: return handleOPTIONS(request);
+                case OPTIONS: return handleOPTIONS();
                 default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
             }
         }
@@ -151,7 +154,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant")) return tenants(request);
         if (path.matches("/application/v4/tenant-pipeline")) return tenantPipelines();
         if (path.matches("/application/v4/athensDomain")) return athensDomains(request);
-        if (path.matches("/application/v4/property")) return properties(request);
+        if (path.matches("/application/v4/property")) return properties();
         if (path.matches("/application/v4/cookiefreshness")) return cookieFreshness(request);
         if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request);
@@ -201,7 +204,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return ErrorResponse.notFoundError("Nothing at " + path);
     }
     
-    private HttpResponse handleOPTIONS(HttpRequest request) {
+    private HttpResponse handleOPTIONS() {
         // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother
         // spelling out the methods supported at each path, which we should
         EmptyJsonResponse response = new EmptyJsonResponse();
@@ -210,8 +213,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
     }
     
     private HttpResponse root(HttpRequest request) {
-        return new ResourceResponse(request,
-                                    "user", "tenant", "tenant-pipeline", "athensDomain", "property", "cookiefreshness");
+        return new ResourceResponse(request, "user", "tenant", "tenant-pipeline", "athensDomain",
+                                    "property", "cookiefreshness");
     }
     
     private HttpResponse authenticatedUser(HttpRequest request) {
@@ -269,7 +272,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         return new SlimeJsonResponse(slime);
     }
 
-    private HttpResponse properties(HttpRequest request) {
+    private HttpResponse properties() {
         Slime slime = new Slime();
         Cursor response = slime.setObject();
         Cursor array = response.setArray("properties");
@@ -323,9 +326,13 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
                 toSlime(((Change.ApplicationChange)application.deploying().get()).revision().get(), deployingObject.setObject("revision"));
         }
 
-        // Deployment jobs 
+        // Jobs sorted according to deployment spec
+        Map<DeploymentJobs.JobType, JobStatus> jobStatus = controller.applications().deploymentTrigger()
+                .deploymentOrder()
+                .sortBy(application.deploymentSpec(), application.deploymentJobs().jobStatus());
+
         Cursor deploymentsArray = response.setArray("deploymentJobs");
-        for (JobStatus job : application.deploymentJobs().jobStatus().values()) {
+        for (JobStatus job : jobStatus.values()) {
             Cursor jobObject = deploymentsArray.addObject();            
             jobObject.setString("type", job.type().id());
             jobObject.setBool("success", job.isSuccess());
@@ -347,9 +354,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         for (URI rotation : rotations)
             globalRotationsArray.addString(rotation.toString());
 
-        // Deployments
+        // Deployments sorted according to deployment spec
+        Map<Zone, Deployment> deployments = controller.applications().deploymentTrigger()
+                .deploymentOrder()
+                .sortBy(application.deploymentSpec().zones(), application.deployments());
         Cursor instancesArray = response.setArray("instances");
-        for (Deployment deployment : application.deployments().values()) {
+        for (Deployment deployment : deployments.values()) {
             Cursor deploymentObject = instancesArray.addObject();
             deploymentObject.setString("environment", deployment.zone().environment().value());
             deploymentObject.setString("region", deployment.zone().region().value());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
index e02a31440ce..03ac073a34a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -5,6 +5,10 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.container.logging.AccessLog;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
@@ -12,6 +16,9 @@ import com.yahoo.vespa.hosted.controller.restapi.Path;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
 import com.yahoo.yolean.Exceptions;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.util.concurrent.Executor;
 import java.util.logging.Level;
 
@@ -21,6 +28,7 @@ import java.util.logging.Level;
  * 
  * @author bratseth
  */
+@SuppressWarnings("unused") // Created by injection
 public class ControllerApiHandler extends LoggingRequestHandler {
 
     private final ControllerMaintenance maintenance;
@@ -34,9 +42,10 @@ public class ControllerApiHandler extends LoggingRequestHandler {
     public HttpResponse handle(HttpRequest request) {
         try {
             switch (request.getMethod()) {
-                case GET: return handleGET(request);
-                case POST: return handlePOST(request);
-                case DELETE: return handleDELETE(request);
+                case GET: return get(request);
+                case POST: return post(request);
+                case DELETE: return delete(request);
+                case PATCH: return patch(request);
                 default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
             }
         }
@@ -49,27 +58,36 @@ public class ControllerApiHandler extends LoggingRequestHandler {
         }
     }
     
-    private HttpResponse handleGET(HttpRequest request) {
+    private HttpResponse get(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/")) return root(request);
         if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl());
-        return ErrorResponse.notFoundError("Nothing at " + path);
+        if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute());
+        return notFound(path);
     }
 
-    private HttpResponse handlePOST(HttpRequest request) {
+    private HttpResponse post(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/maintenance/inactive/{jobName}")) 
             return setActive(path.get("jobName"), false);
-        return ErrorResponse.notFoundError("Nothing at " + path);
+        return notFound(path);
     }
 
-    private HttpResponse handleDELETE(HttpRequest request) {
+    private HttpResponse delete(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/maintenance/inactive/{jobName}"))
             return setActive(path.get("jobName"), true);
-        return ErrorResponse.notFoundError("Nothing at " + path);
+        return notFound(path);
     }
 
+    private HttpResponse patch(HttpRequest request) {
+        Path path = new Path(request.getUri().getPath());
+        if (path.matches("/controller/v1/jobs/upgrader")) return configureUpgrader(request);
+        return notFound(path);
+    }
+
+    private HttpResponse notFound(Path path) { return ErrorResponse.notFoundError("Nothing at " + path); }
+
     private HttpResponse root(HttpRequest request) {
         return new ResourceResponse(request, "maintenance");
     }
@@ -81,4 +99,23 @@ public class ControllerApiHandler extends LoggingRequestHandler {
         return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
     }
 
+    private HttpResponse configureUpgrader(HttpRequest request) {
+        String upgradesPerMinuteField = "upgradesPerMinute";
+        Slime slime = toSlime(request.getData());
+        Inspector inspect = slime.get();
+        if (inspect.field(upgradesPerMinuteField).valid()) {
+            maintenance.upgrader().setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble());
+        }
+        return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute());
+    }
+
+    private Slime toSlime(InputStream jsonStream) {
+        try {
+            byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000);
+            return SlimeUtils.jsonToSlime(jsonBytes);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java
new file mode 100644
index 00000000000..fe88a0f1f22
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java
@@ -0,0 +1,35 @@
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author mpolden
+ */
+public class UpgraderResponse extends HttpResponse {
+
+    private final double upgradesPerMinute;
+
+    public UpgraderResponse(double upgradesPerMinute) {
+        super(200);
+        this.upgradesPerMinute = upgradesPerMinute;
+    }
+
+    @Override
+    public void render(OutputStream outputStream) throws IOException {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        root.setDouble("upgradesPerMinute", upgradesPerMinute);
+        new JsonFormat(true).encode(outputStream, slime);
+    }
+
+    @Override
+    public String getContentType() {
+        return "application/json";
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 4c23c092cc9..db913d8c8d5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -396,64 +396,76 @@ public class ControllerTest {
     public void requeueOutOfCapacityStagingJob() {
         DeploymentTester tester = new DeploymentTester();
 
-        long fooProjectId = 1;
-        long barProjectId = 2;
-        Application foo = tester.createApplication("app1", "foo", fooProjectId, 1L);
-        Application bar = tester.createApplication("app2", "bar", barProjectId, 1L);
+        long project1 = 1;
+        long project2 = 2;
+        long project3 = 3;
+        Application app1 = tester.createApplication("app1", "tenant1", project1, 1L);
+        Application app2 = tester.createApplication("app2", "tenant2", project2, 1L);
+        Application app3 = tester.createApplication("app3", "tenant3", project3, 1L);
         BuildSystem buildSystem = tester.controller().applications().deploymentTrigger().buildSystem();
 
-        // foo: passes system test
-        tester.notifyJobCompletion(component, foo, true);
-        tester.deployAndNotify(foo, applicationPackage, true, systemTest);
+        // all applications: system-test completes successfully
+        tester.notifyJobCompletion(component, app1, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
 
-        // bar: passes system test
-        tester.notifyJobCompletion(component, bar, true);
-        tester.deployAndNotify(bar, applicationPackage, true, systemTest);
+        tester.notifyJobCompletion(component, app2, true);
+        tester.deployAndNotify(app2, applicationPackage, true, systemTest);
 
-        // foo and bar: staging test jobs queued
-        assertEquals(2, buildSystem.jobs().size());
+        tester.notifyJobCompletion(component, app3, true);
+        tester.deployAndNotify(app3, applicationPackage, true, systemTest);
 
-        // foo: staging-test job fails with out of capacity and is added to the front of the queue
-        {
-            tester.deploy(stagingTest, foo, applicationPackage);
-            tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity));
-            List<BuildJob> nextJobs = buildSystem.takeJobsToRun();
-            assertEquals("staging-test jobs are returned one at a time",1, nextJobs.size());
-            assertEquals(stagingTest.id(), nextJobs.get(0).jobName());
-            assertEquals(fooProjectId, nextJobs.get(0).projectId());
-        }
+        // all applications: staging test jobs queued
+        assertEquals(3, buildSystem.jobs().size());
 
-        // bar: Completes deployment
-        tester.deployAndNotify(bar, applicationPackage, true, stagingTest);
-        tester.deployAndNotify(bar, applicationPackage, true, productionCorpUsEast1);
+        // app1: staging-test job fails with out of capacity and is added to the front of the queue
+        tester.deploy(stagingTest, app1, applicationPackage);
+        tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity));
+        assertEquals(stagingTest.id(), buildSystem.jobs().get(0).jobName());
+        assertEquals(project1, buildSystem.jobs().get(0).projectId());
 
-        // foo: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by
+        // app2 and app3: Completes deployment
+        tester.deployAndNotify(app2, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app2, applicationPackage, true, productionCorpUsEast1);
+        tester.deployAndNotify(app3, applicationPackage, true, stagingTest);
+        tester.deployAndNotify(app3, applicationPackage, true, productionCorpUsEast1);
+
+        // app1: 15 minutes pass, staging-test job is still failing due out of capacity, but is no longer re-queued by
         // out of capacity retry mechanism
         tester.clock().advance(Duration.ofMinutes(15));
-        tester.notifyJobCompletion(component, foo, true);
-        tester.deployAndNotify(foo, applicationPackage, true, systemTest);
-        tester.deploy(stagingTest, foo, applicationPackage);
+        tester.notifyJobCompletion(component, app1, true);
+        tester.deployAndNotify(app1, applicationPackage, true, systemTest);
+        tester.deploy(stagingTest, app1, applicationPackage);
         assertEquals(1, buildSystem.takeJobsToRun().size());
-        tester.notifyJobCompletion(stagingTest, foo, Optional.of(JobError.outOfCapacity));
+        tester.notifyJobCompletion(stagingTest, app1, Optional.of(JobError.outOfCapacity));
         assertTrue("No jobs queued", buildSystem.jobs().isEmpty());
 
-        // bar: New change triggers another staging-test job
-        tester.notifyJobCompletion(component, bar, true);
-        tester.deployAndNotify(bar, applicationPackage, true, systemTest);
-        assertEquals(1, buildSystem.jobs().size());
+        // app2 and app3: New change triggers staging-test jobs
+        tester.notifyJobCompletion(component, app2, true);
+        tester.deployAndNotify(app2, applicationPackage, true, systemTest);
 
-        // foo: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the
+        tester.notifyJobCompletion(component, app3, true);
+        tester.deployAndNotify(app3, applicationPackage, true, systemTest);
+
+        assertEquals(2, buildSystem.jobs().size());
+
+        // app1: 4 hours pass in total, staging-test job is re-queued by periodic trigger mechanism and added at the
         // back of the queue
         tester.clock().advance(Duration.ofHours(3));
         tester.clock().advance(Duration.ofMinutes(50));
         tester.failureRedeployer().maintain();
 
         List<BuildJob> nextJobs = buildSystem.takeJobsToRun();
+        assertEquals(2, nextJobs.size());
         assertEquals(stagingTest.id(), nextJobs.get(0).jobName());
-        assertEquals(barProjectId, nextJobs.get(0).projectId());
+        assertEquals(project2, nextJobs.get(0).projectId());
+        assertEquals(stagingTest.id(), nextJobs.get(1).jobName());
+        assertEquals(project3, nextJobs.get(1).projectId());
+
+        // And finally the requeued job for app1
         nextJobs = buildSystem.takeJobsToRun();
+        assertEquals(1, nextJobs.size());
         assertEquals(stagingTest.id(), nextJobs.get(0).jobName());
-        assertEquals(fooProjectId, nextJobs.get(0).projectId());
+        assertEquals(project1, nextJobs.get(0).projectId());
     }
 
     private void assertStatus(JobStatus expectedStatus, ApplicationId id, Controller controller) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 71eb85cd0a4..ad723677686 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -50,7 +50,9 @@ public class DeploymentTester {
 
     public DeploymentTester(ControllerTester tester) {
         this.tester = tester;
-        this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()));
+        tester.curator().writeUpgradesPerMinute(100);
+        this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()),
+                                     tester.curator());
         this.failureRedeployer = new FailureRedeployer(tester.controller(), maintenanceInterval,
                                                        new JobControl(tester.curator()));
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 5a48bc54b49..5abadf28cfb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -53,15 +53,9 @@ public class DeploymentTriggerTest {
         tester.buildSystem().takeJobsToRun();
         assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
         tester.clock().advance(Duration.ofHours(4).plus(Duration.ofSeconds(1)));
-        tester.failureRedeployer().maintain();
+        tester.failureRedeployer().maintain(); // Causes retry of systemTests
 
-        assertEquals("Retried job", 1, tester.buildSystem().jobs().size());
-        assertEquals(JobType.systemTest.id(), tester.buildSystem().jobs().get(0).jobName());
-        tester.buildSystem().takeJobsToRun();
-        assertEquals("Job removed", 0, tester.buildSystem().jobs().size());
-
-        // system-test succeeds and staging-test starts
-        tester.failureRedeployer().maintain();
+        assertEquals("Scheduled retry", 1, tester.buildSystem().jobs().size());
         tester.deployAndNotify(app, applicationPackage, true, JobType.systemTest);
 
         // staging-test times out and is retried
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java
index c869bd90924..e66d7e9168d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/PolledBuildSystemTest.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
 public class PolledBuildSystemTest {
 
     @Parameterized.Parameters(name = "jobType={0}")
-    public static Iterable<? extends Object> capacityConstrainedJobs() {
+    public static Iterable<?> capacityConstrainedJobs() {
         return Arrays.asList(JobType.systemTest, JobType.stagingTest);
     }
 
@@ -37,26 +37,32 @@ public class PolledBuildSystemTest {
         DeploymentTester tester = new DeploymentTester();
         BuildSystem buildSystem = new PolledBuildSystem(tester.controller(), new MockCuratorDb());
 
-        int fooProjectId = 1;
-        int barProjectId = 2;
+        int project1 = 1;
+        int project2 = 2;
+        int project3 = 3;
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .region("us-west-1")
                 .build();
-        ApplicationId foo = tester.createAndDeploy("app1", fooProjectId, applicationPackage).id();
-        ApplicationId bar = tester.createAndDeploy("app2", barProjectId, applicationPackage).id();
+        ApplicationId app1 = tester.createAndDeploy("app1", project1, applicationPackage).id();
+        ApplicationId app2 = tester.createAndDeploy("app2", project2, applicationPackage).id();
+        ApplicationId app3 = tester.createAndDeploy("app3", project3, applicationPackage).id();
 
         // Trigger jobs in capacity constrained environment
-        buildSystem.addJob(foo, jobType, false);
-        buildSystem.addJob(bar, jobType, false);
+        buildSystem.addJob(app1, jobType, false);
+        buildSystem.addJob(app2, jobType, false);
+        buildSystem.addJob(app3, jobType, false);
 
-        // Capacity constrained jobs are returned one a at a time
+        // A limited number of jobs are offered at a time:
+        // First offer
         List<BuildJob> nextJobs = buildSystem.takeJobsToRun();
-        assertEquals(1, nextJobs.size());
-        assertEquals(fooProjectId, nextJobs.get(0).projectId());
+        assertEquals(2, nextJobs.size());
+        assertEquals(project1, nextJobs.get(0).projectId());
+        assertEquals(project2, nextJobs.get(1).projectId());
 
+        // Second offer
         nextJobs = buildSystem.takeJobsToRun();
         assertEquals(1, nextJobs.size());
-        assertEquals(barProjectId, nextJobs.get(0).projectId());
+        assertEquals(project3, nextJobs.get(0).projectId());
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index bc7b017dd7d..95b700f4dd9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -466,4 +466,50 @@ public class UpgraderTest {
                      tester.buildSystem().jobs().get(0).jobName());
     }
 
+    @Test
+    public void testThrottlesUpgrades() {
+        DeploymentTester tester = new DeploymentTester();
+        Version version = Version.fromString("5.0");
+        tester.updateVersionStatus(version);
+
+        // Setup our own upgrader as we need to control the interval
+        Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(10),
+                                         new JobControl(tester.controllerTester().curator()),
+                                         tester.controllerTester().curator());
+        upgrader.setUpgradesPerMinute(0.2);
+
+        // Setup applications
+        Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
+        Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
+        Application default0 = tester.createAndDeploy("default0", 3, "default");
+        Application default1 = tester.createAndDeploy("default1", 4, "default");
+        Application default2 = tester.createAndDeploy("default2", 5, "default");
+        Application default3 = tester.createAndDeploy("default3", 6, "default");
+
+        // New version is released and canaries upgrade
+        version = Version.fromString("5.1");
+        tester.updateVersionStatus(version);
+        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        upgrader.maintain();
+
+        assertEquals(2, tester.buildSystem().jobs().size());
+        tester.completeUpgrade(canary0, version, "canary");
+        tester.completeUpgrade(canary1, version, "canary");
+        tester.updateVersionStatus(version);
+
+        // Next run upgrades a subset
+        upgrader.maintain();
+        assertEquals(2, tester.buildSystem().jobs().size());
+        tester.completeUpgrade(default0, version, "default");
+        tester.completeUpgrade(default2, version, "default");
+
+        // Remaining applications upgraded
+        upgrader.maintain();
+        assertEquals(2, tester.buildSystem().jobs().size());
+        tester.completeUpgrade(default1, version, "default");
+        tester.completeUpgrade(default3, version, "default");
+        upgrader.maintain();
+        assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
index ed7378ac6b5..a792626d691 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
@@ -23,13 +23,14 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
-import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
-import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 
 import java.io.File;
@@ -51,7 +52,9 @@ public class ContainerControllerTester {
     public ContainerControllerTester(JDisc container, String responseFilePath) {
         containerTester = new ContainerTester(container, responseFilePath);
         controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller");
-        upgrader = new Upgrader(controller, Duration.ofMinutes(2), new JobControl(new MockCuratorDb()));
+        CuratorDb curatorDb = new MockCuratorDb();
+        curatorDb.writeUpgradesPerMinute(100);
+        upgrader = new Upgrader(controller, Duration.ofDays(1), new JobControl(curatorDb), curatorDb);
     }
 
     public Controller controller() { return controller; }
@@ -112,4 +115,8 @@ public class ContainerControllerTester {
         containerTester.assertResponse(request, expectedResponse);
     }
 
+    public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException {
+        containerTester.assertResponse(request, expectedResponse, expectedStatusCode);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 18073ef06f3..af05c7bbb58 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -309,6 +309,74 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       athensScrewdriverDomain, "screwdriveruser1"),
                               new File("deploy-result.json"));
     }
+
+    @Test
+    public void testSortsDeploymentsAndJobs() throws Exception {
+        // Setup
+        ContainerControllerTester controllerTester = new ContainerControllerTester(container, responseFiles);
+        ContainerTester tester = controllerTester.containerTester();
+        tester.updateSystemVersion();
+        addTenantAthensDomain(athensUserDomain, "mytenant");
+        addScrewdriverUserToDomain("screwdriveruser1", "domain1");
+
+        // Create tenant
+        tester.assertResponse(request("/application/v4/tenant/tenant1",
+                                      "{\"athensDomain\":\"domain1\", \"property\":\"property1\"}",
+                                      Request.Method.POST),
+                              new File("tenant-without-applications.json"));
+
+        // Create application
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1",
+                                      "",
+                                      Request.Method.POST),
+                              new File("application-reference.json"));
+
+        // Deploy
+        ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
+                .region("us-east-3")
+                .build();
+        ApplicationId id = ApplicationId.from("tenant1", "application1", "default");
+        long projectId = 1;
+        HttpEntity deployData = createApplicationDeployData(applicationPackage, Optional.of(projectId));
+
+        startAndTestChange(controllerTester, id, projectId, deployData);
+
+        // us-east-3
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy",
+                                      deployData,
+                                      Request.Method.POST,
+                                      athensScrewdriverDomain, "screwdriveruser1"),
+                              new File("deploy-result.json"));
+        controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsEast3);
+
+        // New zone is added before us-east-3
+        applicationPackage = new ApplicationPackageBuilder()
+                // These decides the ordering of deploymentJobs and instances in the response
+                .region("us-west-1")
+                .region("us-east-3")
+                .build();
+        deployData = createApplicationDeployData(applicationPackage, Optional.of(projectId));
+        startAndTestChange(controllerTester, id, projectId, deployData);
+
+        // us-west-1
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy",
+                                      deployData,
+                                      Request.Method.POST,
+                                      athensScrewdriverDomain, "screwdriveruser1"),
+                              new File("deploy-result.json"));
+        controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsWest1);
+
+        // us-east-3
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy",
+                                      deployData,
+                                      Request.Method.POST,
+                                      athensScrewdriverDomain, "screwdriveruser1"),
+                              new File("deploy-result.json"));
+        controllerTester.notifyJobCompletion(id, projectId, true, DeploymentJobs.JobType.productionUsEast3);
+
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", "", Request.Method.GET),
+                              new File("application-without-change-multiple-deployments.json"));
+    }
     
     @Test
     public void testErrorResponses() throws Exception {
@@ -643,5 +711,41 @@ public class ApplicationApiTest extends ControllerContainerTest {
         mock.setApplicationCost(new ApplicationId.Builder().tenant(tenant).applicationName(application).instanceName(instance).build(),
                                 new ApplicationCost("prod.us-west-1", tenant, application + "." + instance, 37, 1.0f, 0.0f, clusterCosts));
     }
+
+    private void startAndTestChange(ContainerControllerTester controllerTester, ApplicationId application, long projectId,
+                                    HttpEntity deployData) throws IOException {
+        ContainerTester tester = controllerTester.containerTester();
+
+        // Trigger application change
+        controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.component);
+
+        // system-test
+        String testPath = String.format("/application/v4/tenant/%s/application/%s/environment/test/region/us-east-1/instance/default",
+                                        application.tenant().value(), application.application().value());
+        tester.assertResponse(request(testPath,
+                                      deployData,
+                                      Request.Method.POST,
+                                      athensScrewdriverDomain, "screwdriveruser1"),
+                              new File("deploy-result.json"));
+        tester.assertResponse(request(testPath,
+                                      "",
+                                      Request.Method.DELETE),
+                              "Deactivated " + testPath.replaceFirst("/application/v4/", ""));
+        controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.systemTest);
+
+        // staging
+        String stagingPath = String.format("/application/v4/tenant/%s/application/%s/environment/staging/region/us-east-3/instance/default",
+                                           application.tenant().value(), application.application().value());
+        tester.assertResponse(request(stagingPath,
+                                      deployData,
+                                      Request.Method.POST,
+                                      athensScrewdriverDomain, "screwdriveruser1"),
+                              new File("deploy-result.json"));
+        tester.assertResponse(request(stagingPath,
+                                      "",
+                                      Request.Method.DELETE),
+                              "Deactivated " + stagingPath.replaceFirst("/application/v4/", ""));
+        controllerTester.notifyJobCompletion(application, projectId, true, DeploymentJobs.JobType.stagingTest);
+    }
     
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
new file mode 100644
index 00000000000..a82bdaa454a
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json
@@ -0,0 +1,204 @@
+{
+  "deploymentJobs": [
+    {
+      "type": "component",
+      "success": true,
+      "lastCompleted": {
+        "version": "(ignore)",
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "version": "(ignore)",
+        "at": "(ignore)"
+      }
+    },
+    {
+      "type": "system-test",
+      "success": true,
+      "lastTriggered": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastCompleted": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      }
+    },
+    {
+      "type": "staging-test",
+      "success": true,
+      "lastTriggered": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastCompleted": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      }
+    },
+    {
+      "type": "production-us-west-1",
+      "success": true,
+      "lastTriggered": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastCompleted": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      }
+    },
+    {
+      "type": "production-us-east-3",
+      "success": true,
+      "lastTriggered": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastCompleted": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      },
+      "lastSuccess": {
+        "version": "(ignore)",
+        "revision": {
+          "hash": "(ignore)",
+          "source": {
+            "gitRepository": "repository1",
+            "gitBranch": "master",
+            "gitCommit": "commit1"
+          }
+        },
+        "at": "(ignore)"
+      }
+    }
+  ],
+  "compileVersion": "(ignore)",
+  "globalRotations": [
+    "http://fake-global-rotation-tenant1.application1"
+  ],
+  "instances": [
+    {
+      "environment": "prod",
+      "region": "us-west-1",
+      "instance": "default",
+      "bcpStatus": {
+        "rotationStatus": "IN"
+      },
+      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default"
+    },
+    {
+      "environment": "prod",
+      "region": "us-east-3",
+      "instance": "default",
+      "bcpStatus": {
+        "rotationStatus": "UNKNOWN"
+      },
+      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default"
+    }
+  ],
+  "metrics": {
+    "queryServiceQuality": 0.5,
+    "writeServiceQuality": 0.7
+  }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json
index d1ae5253a00..06b48064b94 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deploy-result.json
@@ -1,6 +1,6 @@
 {
   "revisionId":"(ignore)",
-  "applicationZipSize":412,
+  "applicationZipSize":"(ignore)",
   "prepareMessages":[],
   "configChangeActions":{
     "restart":[],
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index 011bbadb91c..e1c5cdb7742 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
 import org.junit.Test;
 
 import java.io.File;
-import java.io.IOException;
 
 /**
  * @author bratseth
@@ -17,7 +16,7 @@ public class ControllerApiTest extends ControllerContainerTest {
     private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/";
 
     @Test
-    public void testControllerApi() throws IOException {
+    public void testControllerApi() throws Exception {
         ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
 
         tester.assertResponse(new Request("http://localhost:8080/controller/v1/"), new File("root.json"));
@@ -37,4 +36,32 @@ public class ControllerApiTest extends ControllerContainerTest {
                        "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}");
     }
 
+    @Test
+    public void testUpgraderApi() throws Exception {
+        ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+
+        // Get current configuration
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader"),
+                              "{\"upgradesPerMinute\":0.5}",
+                              200);
+
+        // Set invalid configuration
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader",
+                                          "{\"upgradesPerMinute\":-1}", Request.Method.PATCH),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0\"}",
+                              400);
+
+        // Unrecognized fields are ignored
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader",
+                                          "{\"foo\":bar}", Request.Method.PATCH),
+                              "{\"upgradesPerMinute\":0.5}",
+                              200);
+
+        // Set configuration
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader",
+                                          "{\"upgradesPerMinute\":42}", Request.Method.PATCH),
+                              "{\"upgradesPerMinute\":42.0}",
+                              200);
+    }
+
 }