Implement REST API for OS upgrades

14 files changed, 671 insertions, 21 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 3733022766e..e60f76815ea 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -232,8 +232,16 @@ public class Controller extends AbstractComponent {
 
     /** Set the target OS version for infrastructure on cloud in this system */
     public void upgradeOsIn(CloudName cloud, Version version) {
+        if (version.isEmpty()) {
+            throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'");
+        }
         try (Lock lock = curator.lockOsVersions()) {
             Set<OsVersion> versions = new TreeSet<>(curator.readOsVersions());
+            if (versions.stream().anyMatch(osVersion -> osVersion.cloud().equals(cloud) &&
+                                                        osVersion.version().isAfter(version))) {
+                throw new IllegalArgumentException("Cannot downgrade cloud '" + cloud.value() + "' to version " +
+                                                   version.toFullString());
+            }
             versions.removeIf(osVersion -> osVersion.cloud().equals(cloud)); // Only allow a single target per cloud
             versions.add(new OsVersion(version, cloud));
             curator.writeOsVersions(versions);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
index 57364cee049..cae48eba242 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.persistence;
 
+import com.google.inject.Inject;
 import com.yahoo.vespa.curator.mock.MockCurator;
 
 /**
@@ -11,6 +12,7 @@ import com.yahoo.vespa.curator.mock.MockCurator;
 @SuppressWarnings("unused") // injected
 public class MockCuratorDb extends CuratorDb {
 
+    @Inject
     public MockCuratorDb() {
         this("test-controller:2222");
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java
index 7256c283ab7..a36a8d8384f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java
@@ -117,10 +117,11 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase {
     private static boolean isHostedOperatorOperation(Path path, Method method) {
         if (isWhiteListedOperation(path, method)) return false;
         return path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") ||
-                path.matches("/controller/v1/{*}") ||
-                path.matches("/provision/v2/{*}") ||
-                path.matches("/screwdriver/v1/trigger/tenant/{*}") ||
-                path.matches("/zone/v2/{*}");
+               path.matches("/controller/v1/{*}") ||
+               path.matches("/provision/v2/{*}") ||
+               path.matches("/screwdriver/v1/trigger/tenant/{*}") ||
+               path.matches("/os/v1/{*}") ||
+               path.matches("/zone/v2/{*}");
     }
 
     private static boolean isTenantAdminOperation(Path path, Method method) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
new file mode 100644
index 00000000000..32c9e194d56
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java
@@ -0,0 +1,131 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.os;
+
+import com.yahoo.component.Version;
+import com.yahoo.container.jdisc.HttpRequest;
+import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.container.jdisc.LoggingRequestHandler;
+import com.yahoo.io.IOUtils;
+import com.yahoo.restapi.Path;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
+import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse;
+import com.yahoo.vespa.hosted.controller.versions.OsVersion;
+import com.yahoo.yolean.Exceptions;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.util.Set;
+import java.util.logging.Level;
+
+/**
+ * This implements the /os/v1 API which provides operators with information about, and scheduling of OS upgrades for
+ * nodes in the system.
+ *
+ * @author mpolden
+ */
+@SuppressWarnings("unused") // Injected
+public class OsApiHandler extends LoggingRequestHandler {
+
+    private final Controller controller;
+
+    public OsApiHandler(Context ctx, Controller controller) {
+        super(ctx);
+        this.controller = controller;
+    }
+
+    @Override
+    public HttpResponse handle(HttpRequest request) {
+        try {
+            switch (request.getMethod()) {
+                case GET: return get(request);
+                case PATCH: return patch(request);
+                default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported");
+            }
+        } catch (IllegalArgumentException e) {
+            return ErrorResponse.badRequest(Exceptions.toMessageString(e));
+        } catch (RuntimeException e) {
+            log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e);
+            return ErrorResponse.internalServerError(Exceptions.toMessageString(e));
+        }
+    }
+
+    private HttpResponse patch(HttpRequest request) {
+        Path path = new Path(request.getUri().getPath());
+        if (path.matches("/os/v1/")) return new SlimeJsonResponse(setOsVersion(request));
+        return ErrorResponse.notFoundError("Nothing at " + path);
+    }
+
+    private HttpResponse get(HttpRequest request) {
+        Path path = new Path(request.getUri().getPath());
+        if (path.matches("/os/v1/")) return new SlimeJsonResponse(osVersions());
+        return ErrorResponse.notFoundError("Nothing at " + path);
+    }
+
+    private Slime setOsVersion(HttpRequest request) {
+        Slime requestData = toSlime(request.getData());
+        Inspector root = requestData.get();
+        Inspector versionField = root.field("version");
+        Inspector cloudField = root.field("cloud");
+        if (!versionField.valid() || !cloudField.valid()) {
+            throw new IllegalArgumentException("Fields 'version' and 'cloud' are required");
+        }
+
+        CloudName cloud = CloudName.from(cloudField.asString());
+        if (controller.zoneRegistry().zones().all().ids().stream().noneMatch(zone -> cloud.equals(zone.cloud()))) {
+            throw new IllegalArgumentException("Cloud '" + cloud.value() + "' does not exist in this system");
+        }
+
+        Version target;
+        try {
+            target = Version.fromString(versionField.asString());
+        } catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Invalid version '" + versionField.asString() + "'", e);
+        }
+
+        controller.upgradeOsIn(cloud, target);
+        Slime response = new Slime();
+        Cursor cursor = response.setObject();
+        cursor.setString("message", "Set target OS version for cloud '" + cloud.value() + "' to " +
+                                    target.toFullString());
+        return response;
+    }
+
+    private Slime osVersions() {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+        Set<OsVersion> osVersions = controller.osVersions();
+
+        Cursor versions = root.setArray("versions");
+        controller.osVersionStatus().versions().forEach((osVersion, nodes) -> {
+            Cursor currentVersionObject = versions.addObject();
+            currentVersionObject.setString("version", osVersion.version().toFullString());
+            currentVersionObject.setBool("targetVersion", osVersions.contains(osVersion));
+            currentVersionObject.setString("cloud", osVersion.cloud().value());
+            Cursor nodesArray = currentVersionObject.setArray("nodes");
+            nodes.forEach(node -> {
+                Cursor nodeObject = nodesArray.addObject();
+                nodeObject.setString("hostname", node.hostname().value());
+                nodeObject.setString("environment", node.environment().value());
+                nodeObject.setString("region", node.region().value());
+            });
+        });
+
+        return slime;
+    }
+
+    private static Slime toSlime(InputStream json) {
+        try {
+            return SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000));
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index 032076a0e95..cb6d971c2d9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -1,6 +1,7 @@
 // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
+import com.google.common.collect.ImmutableList;
 import com.google.inject.Inject;
 import com.yahoo.component.AbstractComponent;
 import com.yahoo.config.provision.Environment;
@@ -35,6 +36,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
     private List<ZoneId> zones = new ArrayList<>();
     private SystemName system = SystemName.main;
     private UpgradePolicy upgradePolicy = null;
+    private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>();
 
     @Inject
     public ZoneRegistryMock() {
@@ -74,6 +76,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
         return this;
     }
 
+    public ZoneRegistryMock setOsUpgradePolicy(CloudName cloud, UpgradePolicy upgradePolicy) {
+        osUpgradePolicies.put(cloud, upgradePolicy);
+        return this;
+    }
+
     @Override
     public SystemName system() {
         return system;
@@ -95,12 +102,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
 
     @Override
     public UpgradePolicy osUpgradePolicy(CloudName cloud) {
-        return upgradePolicy();
+        return osUpgradePolicies.get(cloud);
     }
 
     @Override
     public List<UpgradePolicy> osUpgradePolicies() {
-        return Collections.singletonList(upgradePolicy());
+        return ImmutableList.copyOf(osUpgradePolicies.values());
     }
 
     @Override
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
index 44823ab5777..045386dd93a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java
@@ -210,7 +210,7 @@ public class OsUpgraderTest {
         tester.controllerTester().zoneRegistry()
               .setZones(zone1, zone2, zone3, zone4, zone5)
               .setSystemName(system)
-              .setUpgradePolicy(upgradePolicy);
+              .setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy);
         return new OsUpgrader(tester.controller(), Duration.ofDays(1),
                               new JobControl(tester.controllerTester().curator()), CloudName.defaultName());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
index 98ed64ba879..97084ea4ae9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
@@ -35,7 +35,7 @@ public class OsVersionStatusUpdaterTest {
         for (ZoneId zone : tester.zoneRegistry().zones().controllerUpgraded().ids()) {
             upgradePolicy = upgradePolicy.upgrade(zone);
         }
-        tester.zoneRegistry().setUpgradePolicy(upgradePolicy);
+        tester.zoneRegistry().setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy);
 
         // Initially empty
         assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
index 60a05e4f938..b571e6f1c48 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java
@@ -8,7 +8,6 @@ import com.yahoo.config.provision.TenantName;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.utils.AthenzIdentities;
 import com.yahoo.vespa.hosted.controller.Application;
-import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.TestIdentities;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
@@ -21,18 +20,18 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 import com.yahoo.vespa.hosted.controller.application.DeploymentJobs;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock;
 import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock;
 import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
+import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
 import com.yahoo.vespa.hosted.controller.maintenance.Upgrader;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
+import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant;
 
 import java.io.File;
-import java.io.IOException;
 import java.time.Duration;
 import java.util.Optional;
 
@@ -96,11 +95,11 @@ public class ContainerControllerTester {
 
     // ---- Delegators:
     
-    public void assertResponse(Request request, File expectedResponse) throws IOException {
+    public void assertResponse(Request request, File expectedResponse) {
         containerTester.assertResponse(request, expectedResponse);
     }
 
-    public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException {
+    public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) {
         containerTester.assertResponse(request, expectedResponse, expectedStatusCode);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
index cc275b0636f..f65cf6af346 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
@@ -110,23 +110,27 @@ public class ContainerTester {
         assertEquals("Status code", expectedStatusCode, response.getStatus());
     }
 
-    public void assertResponse(Supplier<Request> request, String expectedResponse) throws IOException {
+    public void assertResponse(Supplier<Request> request, String expectedResponse) {
         assertResponse(request.get(), expectedResponse, 200);
     }
 
-    public void assertResponse(Request request, String expectedResponse) throws IOException {
+    public void assertResponse(Request request, String expectedResponse) {
         assertResponse(request, expectedResponse, 200);
     }
 
-    public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) throws IOException {
+    public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) {
         assertResponse(request.get(), expectedResponse, expectedStatusCode);
     }
 
-    public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException {
+    public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) {
         FilterResult filterResult = invokeSecurityFilters(request);
         request = filterResult.request;
         Response response = filterResult.response != null ? filterResult.response : container.handleRequest(request);
-        assertEquals(expectedResponse, response.getBodyAsString());
+        try {
+            assertEquals(expectedResponse, response.getBodyAsString());
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
         assertEquals("Status code", expectedStatusCode, response.getStatus());
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
index 978e8bce2f4..55f63c7137b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java
@@ -31,7 +31,8 @@ import static org.junit.Assert.assertEquals;
  */
 public class ControllerContainerTest {
 
-    public static final AthenzUser USER = AthenzUser.fromUserId("bob");
+    private static final AthenzUser defaultUser = AthenzUser.fromUserId("bob");
+
     protected JDisc container;
 
     @Before
@@ -92,6 +93,9 @@ public class ControllerContainerTest {
             "  <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>\n" +
             "    <binding>http://*/controller/v1/*</binding>\n" +
             "  </handler>\n" +
+            "  <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" +
+            "    <binding>http://*/os/v1/*</binding>\n" +
+            "  </handler>\n" +
             "  <handler id='com.yahoo.vespa.hosted.controller.restapi.screwdriver.ScrewdriverApiHandler'>\n" +
             "    <binding>http://*/screwdriver/v1/*</binding>\n" +
             "  </handler>\n" +
@@ -127,11 +131,11 @@ public class ControllerContainerTest {
     }
 
     protected static Request authenticatedRequest(String uri) {
-        return addIdentityToRequest(new Request(uri), USER);
+        return addIdentityToRequest(new Request(uri), defaultUser);
     }
 
     protected static Request authenticatedRequest(String uri, byte[] body, Request.Method method) {
-        return addIdentityToRequest(new Request(uri, body, method), USER);
+        return addIdentityToRequest(new Request(uri, body, method), defaultUser);
     }
 
     protected static Request addIdentityToRequest(Request request, AthenzIdentity identity) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
new file mode 100644
index 00000000000..2d76517bfe7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java
@@ -0,0 +1,157 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.restapi.os;
+
+
+import com.google.common.collect.ImmutableList;
+import com.yahoo.application.container.handler.Request;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.vespa.athenz.api.AthenzIdentity;
+import com.yahoo.vespa.athenz.api.AthenzUser;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
+import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
+import com.yahoo.vespa.hosted.controller.maintenance.JobControl;
+import com.yahoo.vespa.hosted.controller.maintenance.Maintainer;
+import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader;
+import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester;
+import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
+import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
+import org.intellij.lang.annotations.Language;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.time.Duration;
+import java.util.List;
+
+/**
+ * @author mpolden
+ */
+public class OsApiTest extends ControllerContainerTest {
+
+    private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/";
+    private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser");
+    private static final CloudName cloud1 = CloudName.from("cloud1");
+    private static final CloudName cloud2 = CloudName.from("cloud2");
+    private static final ZoneId zone1 = ZoneId.from("prod", "us-east-3", cloud1.value());
+    private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1", cloud1.value());
+    private static final ZoneId zone3 = ZoneId.from("prod", "eu-west-1", cloud2.value());
+
+    private ContainerControllerTester tester;
+    private List<OsUpgrader> osUpgraders;
+
+    @Before
+    public void before() {
+        tester = new ContainerControllerTester(container, responses);
+        addUserToHostedOperatorRole(operator);
+        zoneRegistryMock().setSystemName(SystemName.cd)
+                          .setZones(zone1, zone2, zone3)
+                          .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2))
+                          .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3));
+        osUpgraders = ImmutableList.of(
+                new OsUpgrader(tester.controller(), Duration.ofDays(1),
+                               new JobControl(tester.controller().curator()),
+                               cloud1),
+                new OsUpgrader(tester.controller(), Duration.ofDays(1),
+                               new JobControl(tester.controller().curator()),
+                               cloud2));
+    }
+
+    @Test
+    public void test_api() {
+        // No versions available yet
+        assertResponse(new Request("http://localhost:8080/os/v1/"), "{\"versions\":[]}", 200);
+
+        // All nodes are initially on empty version
+        upgradeAndUpdateStatus();
+        assertFile(new Request("http://localhost:8080/os/v1/"), "versions-initial.json");
+
+        // Upgrade OS to a different version in each cloud
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\"}", Request.Method.PATCH),
+                       "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2\"}", 200);
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"8.2.1\", \"cloud\": \"cloud2\"}", Request.Method.PATCH),
+                       "{\"message\":\"Set target OS version for cloud 'cloud2' to 8.2.1\"}", 200);
+
+        // Status is updated after some zones are upgraded
+        upgradeAndUpdateStatus();
+        completeUpgrade(zone1);
+        assertFile(new Request("http://localhost:8080/os/v1/"), "versions-partially-upgraded.json");
+
+        // All zones are upgraded
+        upgradeAndUpdateStatus();
+        completeUpgrade(zone2, zone3);
+        assertFile(new Request("http://localhost:8080/os/v1/"), "versions-all-upgraded.json");
+
+
+        // Error: Missing field 'cloud' or 'version'
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\"}", Request.Method.PATCH),
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Fields 'version' and 'cloud' are required\"}", 400);
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"cloud\": \"cloud1\"}", Request.Method.PATCH),
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Fields 'version' and 'cloud' are required\"}", 400);
+
+        // Error: Invalid versions
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": null, \"cloud\": \"cloud1\"}", Request.Method.PATCH),
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid version '0.0.0'\"}", 400);
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"foo\", \"cloud\": \"cloud1\"}", Request.Method.PATCH),
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid version 'foo': For input string: \\\"foo\\\"\"}", 400);
+
+        // Error: Invalid cloud
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\", \"cloud\": \"foo\"}", Request.Method.PATCH),
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cloud 'foo' does not exist in this system\"}", 400);
+
+        // Error: Downgrade OS
+        assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.4.1\", \"cloud\": \"cloud1\"}", Request.Method.PATCH),
+                       "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade cloud 'cloud1' to version 7.4.1\"}", 400);
+
+    }
+
+    private void upgradeAndUpdateStatus() {
+        osUpgraders.forEach(Maintainer::run);
+        updateVersionStatus();
+    }
+
+    private void updateVersionStatus() {
+        tester.controller().updateOsVersionStatus(OsVersionStatus.compute(tester.controller()));
+    }
+
+    private void completeUpgrade(ZoneId... zones) {
+        for (ZoneId zone : zones) {
+            for (SystemApplication application : SystemApplication.all()) {
+                for (Node node : nodeRepository().list(zone, application.id())) {
+                    nodeRepository().putByHostname(zone, new Node(
+                            node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(),
+                            node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(),
+                            node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(),
+                            node.wantedRebootGeneration()));
+                }
+            }
+        }
+        updateVersionStatus();
+    }
+
+    private ZoneRegistryMock zoneRegistryMock() {
+        return (ZoneRegistryMock) tester.containerTester().container().components()
+                                        .getComponent(ZoneRegistryMock.class.getName());
+    }
+
+    private NodeRepositoryMock nodeRepository() {
+        return ((ConfigServerMock) tester.containerTester().container().components()
+                                         .getComponent(ConfigServerMock.class.getName())).nodeRepository();
+    }
+
+    private void assertResponse(Request request, @Language("JSON") String body, int statusCode) {
+        addIdentityToRequest(request, operator);
+        tester.assertResponse(request, body, statusCode);
+    }
+
+    private void assertFile(Request request, String filename) {
+        addIdentityToRequest(request, operator);
+        tester.assertResponse(request, new File(filename));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
new file mode 100644
index 00000000000..e1d81a874dc
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json
@@ -0,0 +1,108 @@
+{
+  "versions": [
+    {
+      "version": "7.5.2",
+      "targetVersion": true,
+      "cloud": "cloud1",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        }
+      ]
+    },
+    {
+      "version": "8.2.1",
+      "targetVersion": true,
+      "cloud": "cloud2",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        }
+      ]
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
new file mode 100644
index 00000000000..9c1625fdcd5
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json
@@ -0,0 +1,108 @@
+{
+  "versions": [
+    {
+      "version": "0.0.0",
+      "targetVersion": false,
+      "cloud": "cloud1",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        }
+      ]
+    },
+    {
+      "version": "0.0.0",
+      "targetVersion": false,
+      "cloud": "cloud2",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        }
+      ]
+    }
+  ]
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
new file mode 100644
index 00000000000..48a96b70df7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json
@@ -0,0 +1,121 @@
+{
+  "versions": [
+    {
+      "version": "0.0.0",
+      "targetVersion": false,
+      "cloud": "cloud1",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "us-west-1"
+        }
+      ]
+    },
+    {
+      "version": "7.5.2",
+      "targetVersion": true,
+      "cloud": "cloud1",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "us-east-3"
+        }
+      ]
+    },
+    {
+      "version": "0.0.0",
+      "targetVersion": false,
+      "cloud": "cloud2",
+      "nodes": [
+        {
+          "hostname": "node-3-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-2-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-1-configserver-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-2-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-1-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        },
+        {
+          "hostname": "node-3-proxy-host",
+          "environment": "prod",
+          "region": "eu-west-1"
+        }
+      ]
+    },
+    {
+      "version": "8.2.1",
+      "targetVersion": true,
+      "cloud": "cloud2",
+      "nodes": []
+    }
+  ]
+}