diff options
author | Martin Polden <mpolden@mpolden.no> | 2018-08-23 12:13:45 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2018-08-23 12:13:45 +0200 |
commit | 7854ede299fa3a10b1b34154bc06ec685a960130 (patch) | |
tree | 416094dbe410decabe4b26dd1ef2d22fe225cf00 /controller-server | |
parent | 9804dcf8cfb10c4530fa439f0e806af61be03d81 (diff) | |
parent | 7e4f2982d33af6f3caba9dc07b15e0d62b9a97c0 (diff) |
Merge pull request #6651 from vespa-engine/mpolden/os-upgrade-rest-api
Implement REST API for OS upgrades
Diffstat (limited to 'controller-server')
17 files changed, 692 insertions, 34 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index db2b7955516..fb299b4563f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -232,8 +232,19 @@ public class Controller extends AbstractComponent { /** Set the target OS version for infrastructure on cloud in this system */ public void upgradeOsIn(CloudName cloud, Version version) { + if (version.isEmpty()) { + throw new IllegalArgumentException("Invalid version '" + version.toFullString() + "'"); + } + if (zoneRegistry.zones().all().ids().stream().noneMatch(zone -> cloud.equals(zone.cloud()))) { + throw new IllegalArgumentException("Cloud '" + cloud.value() + "' does not exist in this system"); + } try (Lock lock = curator.lockOsVersions()) { Set<OsVersion> versions = new TreeSet<>(curator.readOsVersions()); + if (versions.stream().anyMatch(osVersion -> osVersion.cloud().equals(cloud) && + osVersion.version().isAfter(version))) { + throw new IllegalArgumentException("Cannot downgrade cloud '" + cloud.value() + "' to version " + + version.toFullString()); + } versions.removeIf(osVersion -> osVersion.cloud().equals(cloud)); // Only allow a single target per cloud versions.add(new OsVersion(version, cloud)); curator.writeOsVersions(versions); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java index 57364cee049..cae48eba242 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/MockCuratorDb.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; +import com.google.inject.Inject; import com.yahoo.vespa.curator.mock.MockCurator; /** @@ -11,6 +12,7 @@ import com.yahoo.vespa.curator.mock.MockCurator; @SuppressWarnings("unused") // injected public class MockCuratorDb extends CuratorDb { + @Inject public MockCuratorDb() { this("test-controller:2222"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java index aee19644b82..0a4a13d3723 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java @@ -47,16 +47,8 @@ public class OsVersionSerializer { public OsVersion fromSlime(Inspector object) { return new OsVersion( Version.fromString(object.field(versionField).asString()), - cloudFrom(object.field(cloudField)) + CloudName.from(object.field(cloudField).asString()) ); } - // TODO: Simplify and inline after 2018-09-01 - private static CloudName cloudFrom(Inspector field) { - if (!field.valid()) { - return CloudName.defaultName(); - } - return CloudName.from(field.asString()); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java index 7256c283ab7..a36a8d8384f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/filter/ControllerAuthorizationFilter.java @@ -117,10 +117,11 @@ public class ControllerAuthorizationFilter extends CorsRequestFilterBase { private static boolean isHostedOperatorOperation(Path path, Method method) { if (isWhiteListedOperation(path, method)) return false; return path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying") || - path.matches("/controller/v1/{*}") || - path.matches("/provision/v2/{*}") || - path.matches("/screwdriver/v1/trigger/tenant/{*}") || - path.matches("/zone/v2/{*}"); + path.matches("/controller/v1/{*}") || + path.matches("/provision/v2/{*}") || + path.matches("/screwdriver/v1/trigger/tenant/{*}") || + path.matches("/os/v1/{*}") || + path.matches("/zone/v2/{*}"); } private static boolean isTenantAdminOperation(Path path, Method method) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java new file mode 100644 index 00000000000..3283fd4fcc4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -0,0 +1,127 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.os; + +import com.yahoo.component.Version; +import com.yahoo.container.jdisc.HttpRequest; +import com.yahoo.container.jdisc.HttpResponse; +import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.io.IOUtils; +import com.yahoo.restapi.Path; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; +import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; +import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; +import com.yahoo.vespa.hosted.controller.versions.OsVersion; +import com.yahoo.yolean.Exceptions; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.util.Set; +import java.util.logging.Level; + +/** + * This implements the /os/v1 API which provides operators with information about, and scheduling of OS upgrades for + * nodes in the system. + * + * @author mpolden + */ +@SuppressWarnings("unused") // Injected +public class OsApiHandler extends LoggingRequestHandler { + + private final Controller controller; + + public OsApiHandler(Context ctx, Controller controller) { + super(ctx); + this.controller = controller; + } + + @Override + public HttpResponse handle(HttpRequest request) { + try { + switch (request.getMethod()) { + case GET: return get(request); + case PATCH: return patch(request); + default: return ErrorResponse.methodNotAllowed("Method '" + request.getMethod() + "' is unsupported"); + } + } catch (IllegalArgumentException e) { + return ErrorResponse.badRequest(Exceptions.toMessageString(e)); + } catch (RuntimeException e) { + log.log(Level.WARNING, "Unexpected error handling '" + request.getUri() + "'", e); + return ErrorResponse.internalServerError(Exceptions.toMessageString(e)); + } + } + + private HttpResponse patch(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/os/v1/")) return new SlimeJsonResponse(setOsVersion(request)); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private HttpResponse get(HttpRequest request) { + Path path = new Path(request.getUri().getPath()); + if (path.matches("/os/v1/")) return new SlimeJsonResponse(osVersions()); + return ErrorResponse.notFoundError("Nothing at " + path); + } + + private Slime setOsVersion(HttpRequest request) { + Slime requestData = toSlime(request.getData()); + Inspector root = requestData.get(); + Inspector versionField = root.field("version"); + Inspector cloudField = root.field("cloud"); + if (!versionField.valid() || !cloudField.valid()) { + throw new IllegalArgumentException("Fields 'version' and 'cloud' are required"); + } + + CloudName cloud = CloudName.from(cloudField.asString()); + Version target; + try { + target = Version.fromString(versionField.asString()); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid version '" + versionField.asString() + "'", e); + } + + controller.upgradeOsIn(cloud, target); + Slime response = new Slime(); + Cursor cursor = response.setObject(); + cursor.setString("message", "Set target OS version for cloud '" + cloud.value() + "' to " + + target.toFullString()); + return response; + } + + private Slime osVersions() { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + Set<OsVersion> osVersions = controller.osVersions(); + + Cursor versions = root.setArray("versions"); + controller.osVersionStatus().versions().forEach((osVersion, nodes) -> { + Cursor currentVersionObject = versions.addObject(); + currentVersionObject.setString("version", osVersion.version().toFullString()); + currentVersionObject.setBool("targetVersion", osVersions.contains(osVersion)); + currentVersionObject.setString("cloud", osVersion.cloud().value()); + Cursor nodesArray = currentVersionObject.setArray("nodes"); + nodes.forEach(node -> { + Cursor nodeObject = nodesArray.addObject(); + nodeObject.setString("hostname", node.hostname().value()); + nodeObject.setString("environment", node.environment().value()); + nodeObject.setString("region", node.region().value()); + }); + }); + + return slime; + } + + private static Slime toSlime(InputStream json) { + try { + return SlimeUtils.jsonToSlime(IOUtils.readBytes(json, 1000 * 1000)); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 5fa442c1b14..f682224e5e9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -35,19 +35,22 @@ import com.yahoo.vespa.hosted.controller.integration.ApplicationStoreMock; import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.MetricsServiceMock; +import com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.persistence.ApplicationSerializer; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.integration.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import java.util.Arrays; import java.util.Optional; import java.util.OptionalLong; +import java.util.function.Consumer; import java.util.function.Supplier; +import java.util.logging.Handler; import java.util.logging.Logger; import static org.junit.Assert.assertNotNull; @@ -128,11 +131,19 @@ public final class ControllerTester { metricsService, routingGenerator); // Make root logger use time from manual clock - Logger.getLogger("").getHandlers()[0].setFilter( + configureDefaultLogHandler(handler -> handler.setFilter( record -> { record.setMillis(clock.millis()); return true; - }); + })); + } + + public void configureDefaultLogHandler(Consumer<Handler> configureFunc) { + Arrays.stream(Logger.getLogger("").getHandlers()) + // Do not mess with log configuration if a custom one has been set + .filter(ignored -> System.getProperty("java.util.logging.config.file") == null) + .findFirst() + .ifPresent(configureFunc); } public static BuildService.BuildJob buildJob(Application application, JobType jobType) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 42d4beedb0b..6eda7909eb5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -89,7 +89,7 @@ public class InternalStepRunnerTest { // Get deployment job logs to stderr. Logger.getLogger(InternalStepRunner.class.getName()).setLevel(DEBUG); Logger.getLogger("").setLevel(DEBUG); - Logger.getLogger("").getHandlers()[0].setLevel(DEBUG); + tester.controllerTester().configureDefaultLogHandler(handler -> handler.setLevel(DEBUG)); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index a13d9f7a19c..cb6d971c2d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -1,11 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import com.google.common.collect.ImmutableList; import com.google.inject.Inject; import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -34,6 +36,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private List<ZoneId> zones = new ArrayList<>(); private SystemName system = SystemName.main; private UpgradePolicy upgradePolicy = null; + private Map<CloudName, UpgradePolicy> osUpgradePolicies = new HashMap<>(); @Inject public ZoneRegistryMock() { @@ -73,6 +76,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } + public ZoneRegistryMock setOsUpgradePolicy(CloudName cloud, UpgradePolicy upgradePolicy) { + osUpgradePolicies.put(cloud, upgradePolicy); + return this; + } + @Override public SystemName system() { return system; @@ -93,8 +101,13 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry } @Override - public UpgradePolicy osUpgradePolicy() { - return upgradePolicy; + public UpgradePolicy osUpgradePolicy(CloudName cloud) { + return osUpgradePolicies.get(cloud); + } + + @Override + public List<UpgradePolicy> osUpgradePolicies() { + return ImmutableList.copyOf(osUpgradePolicies.values()); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 44823ab5777..045386dd93a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -210,7 +210,7 @@ public class OsUpgraderTest { tester.controllerTester().zoneRegistry() .setZones(zone1, zone2, zone3, zone4, zone5) .setSystemName(system) - .setUpgradePolicy(upgradePolicy); + .setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy); return new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.controllerTester().curator()), CloudName.defaultName()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index 98ed64ba879..97084ea4ae9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -35,7 +35,7 @@ public class OsVersionStatusUpdaterTest { for (ZoneId zone : tester.zoneRegistry().zones().controllerUpgraded().ids()) { upgradePolicy = upgradePolicy.upgrade(zone); } - tester.zoneRegistry().setUpgradePolicy(upgradePolicy); + tester.zoneRegistry().setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy); // Initially empty assertSame(OsVersionStatus.empty, tester.controller().osVersionStatus()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 60a05e4f938..b571e6f1c48 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -8,7 +8,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.utils.AthenzIdentities; import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.TestIdentities; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; @@ -21,18 +20,18 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; +import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Upgrader; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; +import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import java.io.File; -import java.io.IOException; import java.time.Duration; import java.util.Optional; @@ -96,11 +95,11 @@ public class ContainerControllerTester { // ---- Delegators: - public void assertResponse(Request request, File expectedResponse) throws IOException { + public void assertResponse(Request request, File expectedResponse) { containerTester.assertResponse(request, expectedResponse); } - public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException { + public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) { containerTester.assertResponse(request, expectedResponse, expectedStatusCode); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index cc275b0636f..f65cf6af346 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -110,23 +110,27 @@ public class ContainerTester { assertEquals("Status code", expectedStatusCode, response.getStatus()); } - public void assertResponse(Supplier<Request> request, String expectedResponse) throws IOException { + public void assertResponse(Supplier<Request> request, String expectedResponse) { assertResponse(request.get(), expectedResponse, 200); } - public void assertResponse(Request request, String expectedResponse) throws IOException { + public void assertResponse(Request request, String expectedResponse) { assertResponse(request, expectedResponse, 200); } - public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) throws IOException { + public void assertResponse(Supplier<Request> request, String expectedResponse, int expectedStatusCode) { assertResponse(request.get(), expectedResponse, expectedStatusCode); } - public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) throws IOException { + public void assertResponse(Request request, String expectedResponse, int expectedStatusCode) { FilterResult filterResult = invokeSecurityFilters(request); request = filterResult.request; Response response = filterResult.response != null ? filterResult.response : container.handleRequest(request); - assertEquals(expectedResponse, response.getBodyAsString()); + try { + assertEquals(expectedResponse, response.getBodyAsString()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } assertEquals("Status code", expectedStatusCode, response.getStatus()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index f7f6927236e..11286acda3a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -31,7 +31,8 @@ import static org.junit.Assert.assertEquals; */ public class ControllerContainerTest { - public static final AthenzUser USER = AthenzUser.fromUserId("bob"); + private static final AthenzUser defaultUser = AthenzUser.fromUserId("bob"); + protected JDisc container; @Before @@ -92,6 +93,9 @@ public class ControllerContainerTest { " <handler id='com.yahoo.vespa.hosted.controller.restapi.controller.ControllerApiHandler'>\n" + " <binding>http://*/controller/v1/*</binding>\n" + " </handler>\n" + + " <handler id='com.yahoo.vespa.hosted.controller.restapi.os.OsApiHandler'>\n" + + " <binding>http://*/os/v1/*</binding>\n" + + " </handler>\n" + " <handler id='com.yahoo.vespa.hosted.controller.restapi.screwdriver.ScrewdriverApiHandler'>\n" + " <binding>http://*/screwdriver/v1/*</binding>\n" + " </handler>\n" + @@ -127,11 +131,11 @@ public class ControllerContainerTest { } protected static Request authenticatedRequest(String uri) { - return addIdentityToRequest(new Request(uri), USER); + return addIdentityToRequest(new Request(uri), defaultUser); } protected static Request authenticatedRequest(String uri, byte[] body, Request.Method method) { - return addIdentityToRequest(new Request(uri, body, method), USER); + return addIdentityToRequest(new Request(uri, body, method), defaultUser); } protected static Request addIdentityToRequest(Request request, AthenzIdentity identity) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java new file mode 100644 index 00000000000..2d76517bfe7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -0,0 +1,157 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.os; + + +import com.google.common.collect.ImmutableList; +import com.yahoo.application.container.handler.Request; +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.athenz.api.AthenzIdentity; +import com.yahoo.vespa.athenz.api.AthenzUser; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; +import com.yahoo.vespa.hosted.controller.api.integration.zone.CloudName; +import com.yahoo.vespa.hosted.controller.api.integration.zone.UpgradePolicy; +import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; +import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; +import com.yahoo.vespa.hosted.controller.maintenance.JobControl; +import com.yahoo.vespa.hosted.controller.maintenance.Maintainer; +import com.yahoo.vespa.hosted.controller.maintenance.OsUpgrader; +import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; +import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; +import org.intellij.lang.annotations.Language; +import org.junit.Before; +import org.junit.Test; + +import java.io.File; +import java.time.Duration; +import java.util.List; + +/** + * @author mpolden + */ +public class OsApiTest extends ControllerContainerTest { + + private static final String responses = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/"; + private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser"); + private static final CloudName cloud1 = CloudName.from("cloud1"); + private static final CloudName cloud2 = CloudName.from("cloud2"); + private static final ZoneId zone1 = ZoneId.from("prod", "us-east-3", cloud1.value()); + private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1", cloud1.value()); + private static final ZoneId zone3 = ZoneId.from("prod", "eu-west-1", cloud2.value()); + + private ContainerControllerTester tester; + private List<OsUpgrader> osUpgraders; + + @Before + public void before() { + tester = new ContainerControllerTester(container, responses); + addUserToHostedOperatorRole(operator); + zoneRegistryMock().setSystemName(SystemName.cd) + .setZones(zone1, zone2, zone3) + .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2)) + .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3)); + osUpgraders = ImmutableList.of( + new OsUpgrader(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator()), + cloud1), + new OsUpgrader(tester.controller(), Duration.ofDays(1), + new JobControl(tester.controller().curator()), + cloud2)); + } + + @Test + public void test_api() { + // No versions available yet + assertResponse(new Request("http://localhost:8080/os/v1/"), "{\"versions\":[]}", 200); + + // All nodes are initially on empty version + upgradeAndUpdateStatus(); + assertFile(new Request("http://localhost:8080/os/v1/"), "versions-initial.json"); + + // Upgrade OS to a different version in each cloud + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.5.2\", \"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud1' to 7.5.2\"}", 200); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"8.2.1\", \"cloud\": \"cloud2\"}", Request.Method.PATCH), + "{\"message\":\"Set target OS version for cloud 'cloud2' to 8.2.1\"}", 200); + + // Status is updated after some zones are upgraded + upgradeAndUpdateStatus(); + completeUpgrade(zone1); + assertFile(new Request("http://localhost:8080/os/v1/"), "versions-partially-upgraded.json"); + + // All zones are upgraded + upgradeAndUpdateStatus(); + completeUpgrade(zone2, zone3); + assertFile(new Request("http://localhost:8080/os/v1/"), "versions-all-upgraded.json"); + + + // Error: Missing field 'cloud' or 'version' + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Fields 'version' and 'cloud' are required\"}", 400); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Fields 'version' and 'cloud' are required\"}", 400); + + // Error: Invalid versions + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": null, \"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid version '0.0.0'\"}", 400); + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"foo\", \"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Invalid version 'foo': For input string: \\\"foo\\\"\"}", 400); + + // Error: Invalid cloud + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.6\", \"cloud\": \"foo\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cloud 'foo' does not exist in this system\"}", 400); + + // Error: Downgrade OS + assertResponse(new Request("http://localhost:8080/os/v1/", "{\"version\": \"7.4.1\", \"cloud\": \"cloud1\"}", Request.Method.PATCH), + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Cannot downgrade cloud 'cloud1' to version 7.4.1\"}", 400); + + } + + private void upgradeAndUpdateStatus() { + osUpgraders.forEach(Maintainer::run); + updateVersionStatus(); + } + + private void updateVersionStatus() { + tester.controller().updateOsVersionStatus(OsVersionStatus.compute(tester.controller())); + } + + private void completeUpgrade(ZoneId... zones) { + for (ZoneId zone : zones) { + for (SystemApplication application : SystemApplication.all()) { + for (Node node : nodeRepository().list(zone, application.id())) { + nodeRepository().putByHostname(zone, new Node( + node.hostname(), node.state(), node.type(), node.owner(), node.currentVersion(), + node.wantedVersion(), node.wantedOsVersion(), node.wantedOsVersion(), node.serviceState(), + node.restartGeneration(), node.wantedRestartGeneration(), node.rebootGeneration(), + node.wantedRebootGeneration())); + } + } + } + updateVersionStatus(); + } + + private ZoneRegistryMock zoneRegistryMock() { + return (ZoneRegistryMock) tester.containerTester().container().components() + .getComponent(ZoneRegistryMock.class.getName()); + } + + private NodeRepositoryMock nodeRepository() { + return ((ConfigServerMock) tester.containerTester().container().components() + .getComponent(ConfigServerMock.class.getName())).nodeRepository(); + } + + private void assertResponse(Request request, @Language("JSON") String body, int statusCode) { + addIdentityToRequest(request, operator); + tester.assertResponse(request, body, statusCode); + } + + private void assertFile(Request request, String filename) { + addIdentityToRequest(request, operator); + tester.assertResponse(request, new File(filename)); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json new file mode 100644 index 00000000000..e1d81a874dc --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -0,0 +1,108 @@ +{ + "versions": [ + { + "version": "7.5.2", + "targetVersion": true, + "cloud": "cloud1", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "us-west-1" + } + ] + }, + { + "version": "8.2.1", + "targetVersion": true, + "cloud": "cloud2", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "eu-west-1" + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json new file mode 100644 index 00000000000..9c1625fdcd5 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json @@ -0,0 +1,108 @@ +{ + "versions": [ + { + "version": "0.0.0", + "targetVersion": false, + "cloud": "cloud1", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "us-west-1" + } + ] + }, + { + "version": "0.0.0", + "targetVersion": false, + "cloud": "cloud2", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "eu-west-1" + } + ] + } + ] +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json new file mode 100644 index 00000000000..48a96b70df7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -0,0 +1,121 @@ +{ + "versions": [ + { + "version": "0.0.0", + "targetVersion": false, + "cloud": "cloud1", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "us-west-1" + } + ] + }, + { + "version": "7.5.2", + "targetVersion": true, + "cloud": "cloud1", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "us-east-3" + } + ] + }, + { + "version": "0.0.0", + "targetVersion": false, + "cloud": "cloud2", + "nodes": [ + { + "hostname": "node-3-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-2-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-proxy-host", + "environment": "prod", + "region": "eu-west-1" + } + ] + }, + { + "version": "8.2.1", + "targetVersion": true, + "cloud": "cloud2", + "nodes": [] + } + ] +} |