Throttle upgrades

10 files changed, 226 insertions, 21 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
index ab1b5aa3cd4..a04eee573c0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/ApplicationList.java
@@ -116,6 +116,12 @@ public class ApplicationList {
         return listOf(list.stream().filter(a -> a.deploymentSpec().canUpgradeAt(instant)));
     }
 
+    /** Returns the first n application in this (or all, if there are less than n). */
+    public ApplicationList first(int n) {
+        if (list.size() < n) return this;
+        return new ApplicationList(list.subList(0, n));
+    }
+
      // ----------------------------------- Sorting
 
     /**
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 016ea66cb1a..1b692ecf243 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.Issues;
 import com.yahoo.vespa.hosted.controller.api.integration.Properties;
 import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef;
 import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
 import java.time.Duration;
 
@@ -33,7 +34,7 @@ public class ControllerMaintenance extends AbstractComponent {
     private final DelayedDeployer delayedDeployer;
 
     @SuppressWarnings("unused") // instantiated by Dependency Injection
-    public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller,
+    public ControllerMaintenance(MaintainerConfig maintainerConfig, Controller controller, CuratorDb curator,
                                  JobControl jobControl, Metric metric, Chef chefClient,
                                  Contacts contactsClient, Properties propertiesClient, Issues issuesClient) {
         Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes());
@@ -45,9 +46,11 @@ public class ControllerMaintenance extends AbstractComponent {
         failureRedeployer = new FailureRedeployer(controller, maintenanceInterval, jobControl);
         outstandingChangeDeployer = new OutstandingChangeDeployer(controller, maintenanceInterval, jobControl);
         versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(3), jobControl);
-        upgrader = new Upgrader(controller, maintenanceInterval, jobControl);
+        upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator);
         delayedDeployer = new DelayedDeployer(controller, maintenanceInterval, jobControl);
     }
+
+    public Upgrader upgrader() { return upgrader; }
     
     /** Returns control of the maintenance jobs of this */
     public JobControl jobControl() { return jobControl; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index cd1d724de9d..9b4ca083df2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -7,6 +7,7 @@ import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.application.ApplicationList;
 import com.yahoo.vespa.hosted.controller.application.Change;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.yolean.Exceptions;
 
@@ -27,8 +28,11 @@ public class Upgrader extends Maintainer {
 
     private static final Logger log = Logger.getLogger(Upgrader.class.getName());
 
-    public Upgrader(Controller controller, Duration interval, JobControl jobControl) {
+    private final CuratorDb curator;
+
+    public Upgrader(Controller controller, Duration interval, JobControl jobControl, CuratorDb curator) {
         super(controller, interval, jobControl);
+        this.curator = curator;
     }
 
     /**
@@ -38,7 +42,7 @@ public class Upgrader extends Maintainer {
     public void maintain() {
         VespaVersion target = controller().versionStatus().version(controller().systemVersion());
         if (target == null) return; // we don't have information about the current system version at this time
-        
+
         switch (target.confidence()) {
             case broken:
                 ApplicationList toCancel = applications().upgradingTo(target.versionNumber())
@@ -74,7 +78,9 @@ public class Upgrader extends Maintainer {
         applications = applications.notFailingOn(version); // try to upgrade only if it hasn't failed on this version
         applications = applications.notCurrentlyUpgrading(change, startOfUpgradePeriod); // do not trigger again if currently upgrading
         applications = applications.canUpgradeAt(controller().clock().instant()); // wait with applications that are currently blocking upgrades
-        for (Application application : applications.byIncreasingDeployedVersion().asList()) {
+        applications = applications.byIncreasingDeployedVersion(); // start with lowest versions
+        applications = applications.first(numberOfApplicationsToUpgrade()); // throttle upgrades
+        for (Application application : applications.asList()) {
             try {
                 controller().applications().deploymentTrigger().triggerChange(application.id(), change);
             } catch (IllegalArgumentException e) {
@@ -89,4 +95,19 @@ public class Upgrader extends Maintainer {
         }
     }
 
+    /** Returns the number of applications to upgrade in this run */
+    private int numberOfApplicationsToUpgrade() {
+        return Math.max(1, (int)(maintenanceInterval().getSeconds() * (upgradesPerMinute() / 60)));
+    }
+
+    /** Returns number upgrades per minute */
+    public double upgradesPerMinute() {
+        return curator.readUpgradesPerMinute();
+    }
+
+    /** Sets the number upgrades per minute */
+    public void setUpgradesPerMinute(double n) {
+        curator.writeUpgradesPerMinute(n);
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index a83a24764ce..66821b22458 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.zookeeper.ZooKeeperServer;
 
+import java.nio.ByteBuffer;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
 import java.util.ArrayDeque;
@@ -192,6 +193,23 @@ public class CuratorDb {
         transaction.commit();
     }
 
+    public double readUpgradesPerMinute() {
+        Optional<byte[]> n = curator.getData(upgradePerMinutePath());
+        if (!n.isPresent() || n.get().length == 0) {
+            return 0.5; // Default if value has never been written
+        }
+        return ByteBuffer.wrap(n.get()).getDouble();
+    }
+
+    public void writeUpgradesPerMinute(double n) {
+        if (n < 0) {
+            throw new IllegalArgumentException("Upgrades per minute must be >= 0");
+        }
+        NestedTransaction transaction = new NestedTransaction();
+        curator.set(upgradePerMinutePath(), ByteBuffer.allocate(Double.BYTES).putDouble(n).array());
+        transaction.commit();
+    }
+
     // -------------- Paths --------------------------------------------------
 
     private Path systemVersionPath() {
@@ -222,4 +240,8 @@ public class CuratorDb {
         return root.append("jobQueues").append(jobType.name());
     }
 
+    private Path upgradePerMinutePath() {
+        return root.append("upgrader").append("upgradesPerMinute");
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
index 2230a36ecec..03ac073a34a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java
@@ -5,6 +5,10 @@ import com.yahoo.container.jdisc.HttpRequest;
 import com.yahoo.container.jdisc.HttpResponse;
 import com.yahoo.container.jdisc.LoggingRequestHandler;
 import com.yahoo.container.logging.AccessLog;
+import com.yahoo.io.IOUtils;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.maintenance.ControllerMaintenance;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
@@ -12,6 +16,9 @@ import com.yahoo.vespa.hosted.controller.restapi.Path;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
 import com.yahoo.yolean.Exceptions;
 
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
 import java.util.concurrent.Executor;
 import java.util.logging.Level;
 
@@ -35,9 +42,10 @@ public class ControllerApiHandler extends LoggingRequestHandler {
     public HttpResponse handle(HttpRequest request) {
         try {
             switch (request.getMethod()) {
-                case GET: return handleGET(request);
-                case POST: return handlePOST(request);
-                case DELETE: return handleDELETE(request);
+                case GET: return get(request);
+                case POST: return post(request);
+                case DELETE: return delete(request);
+                case PATCH: return patch(request);
                 default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is not supported");
             }
         }
@@ -50,27 +58,36 @@ public class ControllerApiHandler extends LoggingRequestHandler {
         }
     }
     
-    private HttpResponse handleGET(HttpRequest request) {
+    private HttpResponse get(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/")) return root(request);
         if (path.matches("/controller/v1/maintenance/")) return new JobsResponse(maintenance.jobControl());
-        return ErrorResponse.notFoundError("Nothing at " + path);
+        if (path.matches("/controller/v1/jobs/upgrader")) return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute());
+        return notFound(path);
     }
 
-    private HttpResponse handlePOST(HttpRequest request) {
+    private HttpResponse post(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/maintenance/inactive/{jobName}")) 
             return setActive(path.get("jobName"), false);
-        return ErrorResponse.notFoundError("Nothing at " + path);
+        return notFound(path);
     }
 
-    private HttpResponse handleDELETE(HttpRequest request) {
+    private HttpResponse delete(HttpRequest request) {
         Path path = new Path(request.getUri().getPath());
         if (path.matches("/controller/v1/maintenance/inactive/{jobName}"))
             return setActive(path.get("jobName"), true);
-        return ErrorResponse.notFoundError("Nothing at " + path);
+        return notFound(path);
     }
 
+    private HttpResponse patch(HttpRequest request) {
+        Path path = new Path(request.getUri().getPath());
+        if (path.matches("/controller/v1/jobs/upgrader")) return configureUpgrader(request);
+        return notFound(path);
+    }
+
+    private HttpResponse notFound(Path path) { return ErrorResponse.notFoundError("Nothing at " + path); }
+
     private HttpResponse root(HttpRequest request) {
         return new ResourceResponse(request, "maintenance");
     }
@@ -82,4 +99,23 @@ public class ControllerApiHandler extends LoggingRequestHandler {
         return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'");
     }
 
+    private HttpResponse configureUpgrader(HttpRequest request) {
+        String upgradesPerMinuteField = "upgradesPerMinute";
+        Slime slime = toSlime(request.getData());
+        Inspector inspect = slime.get();
+        if (inspect.field(upgradesPerMinuteField).valid()) {
+            maintenance.upgrader().setUpgradesPerMinute(inspect.field(upgradesPerMinuteField).asDouble());
+        }
+        return new UpgraderResponse(maintenance.upgrader().upgradesPerMinute());
+    }
+
+    private Slime toSlime(InputStream jsonStream) {
+        try {
+            byte[] jsonBytes = IOUtils.readBytes(jsonStream, 1000 * 1000);
+            return SlimeUtils.jsonToSlime(jsonBytes);
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java
new file mode 100644
index 00000000000..fe88a0f1f22
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/UpgraderResponse.java
@@ -0,0 +1,35 @@
+package com.yahoo.vespa.hosted.controller.restapi.controller;
+
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.Slime;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * @author mpolden
+ */
+public class UpgraderResponse extends HttpResponse {
+
+    private final double upgradesPerMinute;
+
+    public UpgraderResponse(double upgradesPerMinute) {
+        super(200);
+        this.upgradesPerMinute = upgradesPerMinute;
+    }
+
+    @Override
+    public void render(OutputStream outputStream) throws IOException {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        root.setDouble("upgradesPerMinute", upgradesPerMinute);
+        new JsonFormat(true).encode(outputStream, slime);
+    }
+
+    @Override
+    public String getContentType() {
+        return "application/json";
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
index 71eb85cd0a4..ad723677686 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java
@@ -50,7 +50,9 @@ public class DeploymentTester {
 
     public DeploymentTester(ControllerTester tester) {
         this.tester = tester;
-        this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()));
+        tester.curator().writeUpgradesPerMinute(100);
+        this.upgrader = new Upgrader(tester.controller(), maintenanceInterval, new JobControl(tester.curator()),
+                                     tester.curator());
         this.failureRedeployer = new FailureRedeployer(tester.controller(), maintenanceInterval,
                                                        new JobControl(tester.curator()));
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
index bc7b017dd7d..95b700f4dd9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java
@@ -466,4 +466,50 @@ public class UpgraderTest {
                      tester.buildSystem().jobs().get(0).jobName());
     }
 
+    @Test
+    public void testThrottlesUpgrades() {
+        DeploymentTester tester = new DeploymentTester();
+        Version version = Version.fromString("5.0");
+        tester.updateVersionStatus(version);
+
+        // Setup our own upgrader as we need to control the interval
+        Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(10),
+                                         new JobControl(tester.controllerTester().curator()),
+                                         tester.controllerTester().curator());
+        upgrader.setUpgradesPerMinute(0.2);
+
+        // Setup applications
+        Application canary0 = tester.createAndDeploy("canary0", 1, "canary");
+        Application canary1 = tester.createAndDeploy("canary1", 2, "canary");
+        Application default0 = tester.createAndDeploy("default0", 3, "default");
+        Application default1 = tester.createAndDeploy("default1", 4, "default");
+        Application default2 = tester.createAndDeploy("default2", 5, "default");
+        Application default3 = tester.createAndDeploy("default3", 6, "default");
+
+        // New version is released and canaries upgrade
+        version = Version.fromString("5.1");
+        tester.updateVersionStatus(version);
+        assertEquals(version, tester.controller().versionStatus().systemVersion().get().versionNumber());
+        upgrader.maintain();
+
+        assertEquals(2, tester.buildSystem().jobs().size());
+        tester.completeUpgrade(canary0, version, "canary");
+        tester.completeUpgrade(canary1, version, "canary");
+        tester.updateVersionStatus(version);
+
+        // Next run upgrades a subset
+        upgrader.maintain();
+        assertEquals(2, tester.buildSystem().jobs().size());
+        tester.completeUpgrade(default0, version, "default");
+        tester.completeUpgrade(default2, version, "default");
+
+        // Remaining applications upgraded
+        upgrader.maintain();
+        assertEquals(2, tester.buildSystem().jobs().size());
+        tester.completeUpgrade(default1, version, "default");
+        tester.completeUpgrade(default3, version, "default");
+        upgrader.maintain();
+        assertTrue("All jobs consumed", tester.buildSystem().jobs().isEmpty());
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
index ed7378ac6b5..a792626d691 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
@@ -23,13 +23,14 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.UserId;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.Athens;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.AthensPrincipal;
-import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
-import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensDbMock;
+import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.AthensMock;
 import com.yahoo.vespa.hosted.controller.api.integration.athens.mock.ZmsClientFactoryMock;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 
 import java.io.File;
@@ -51,7 +52,9 @@ public class ContainerControllerTester {
     public ContainerControllerTester(JDisc container, String responseFilePath) {
         containerTester = new ContainerTester(container, responseFilePath);
         controller = (Controller)container.components().getComponent("com.yahoo.vespa.hosted.controller.Controller");
-        upgrader = new Upgrader(controller, Duration.ofMinutes(2), new JobControl(new MockCuratorDb()));
+        CuratorDb curatorDb = new MockCuratorDb();
+        curatorDb.writeUpgradesPerMinute(100);
+        upgrader = new Upgrader(controller, Duration.ofDays(1), new JobControl(curatorDb), curatorDb);
     }
 
     public Controller controller() { return controller; }
@@ -112,4 +115,8 @@ public class ContainerControllerTester {
         containerTester.assertResponse(request, expectedResponse);
     }
 
+    public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException {
+        containerTester.assertResponse(request, expectedResponse, expectedStatusCode);
+    }
+
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index 011bbadb91c..e1c5cdb7742 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -7,7 +7,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
 import org.junit.Test;
 
 import java.io.File;
-import java.io.IOException;
 
 /**
  * @author bratseth
@@ -17,7 +16,7 @@ public class ControllerApiTest extends ControllerContainerTest {
     private final static String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/";
 
     @Test
-    public void testControllerApi() throws IOException {
+    public void testControllerApi() throws Exception {
         ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
 
         tester.assertResponse(new Request("http://localhost:8080/controller/v1/"), new File("root.json"));
@@ -37,4 +36,32 @@ public class ControllerApiTest extends ControllerContainerTest {
                        "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}");
     }
 
+    @Test
+    public void testUpgraderApi() throws Exception {
+        ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles);
+
+        // Get current configuration
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader"),
+                              "{\"upgradesPerMinute\":0.5}",
+                              200);
+
+        // Set invalid configuration
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader",
+                                          "{\"upgradesPerMinute\":-1}", Request.Method.PATCH),
+                              "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Upgrades per minute must be >= 0\"}",
+                              400);
+
+        // Unrecognized fields are ignored
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader",
+                                          "{\"foo\":bar}", Request.Method.PATCH),
+                              "{\"upgradesPerMinute\":0.5}",
+                              200);
+
+        // Set configuration
+        tester.assertResponse(new Request("http://localhost:8080/controller/v1/jobs/upgrader",
+                                          "{\"upgradesPerMinute\":42}", Request.Method.PATCH),
+                              "{\"upgradesPerMinute\":42.0}",
+                              200);
+    }
+
 }