From ac27bc5e408d13976214355884adff5881c24b68 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 10 May 2019 12:36:06 +0200 Subject: Allow alwaysRun steps to fail --- .../controller/deployment/InternalStepRunner.java | 28 ++++++++++++++++------ .../vespa/hosted/controller/deployment/Run.java | 8 +++---- .../vespa/hosted/controller/deployment/Step.java | 2 +- 3 files changed, 26 insertions(+), 12 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 472a0c5fb7e..fdadae065d2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -451,20 +451,33 @@ public class InternalStepRunner implements StepRunner { } catch (Exception e) { logger.log(INFO, "Failure getting vespa logs for " + id, e); + return Optional.of(error); } - return Optional.of(running); // Don't let failure here stop cleanup. + return Optional.of(running); } private Optional deactivateReal(RunId id, DualLogger logger) { - logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.applications().deactivate(id.application(), id.type().zone(controller.system())); - return Optional.of(running); + try { + logger.log("Deactivating deployment of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.applications().deactivate(id.application(), id.type().zone(controller.system())); + return Optional.of(running); + } + catch (RuntimeException e) { + logger.log(WARNING, "Failed deleting application " + id.application(), e); + return Optional.of(error); + } } private Optional deactivateTester(RunId id, DualLogger logger) { - logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); - controller.jobController().deactivateTester(id.tester(), id.type()); - return Optional.of(running); + try { + logger.log("Deactivating tester of " + id.application() + " in " + id.type().zone(controller.system()) + " ..."); + controller.jobController().deactivateTester(id.tester(), id.type()); + return Optional.of(running); + } + catch (RuntimeException e) { + logger.log(WARNING, "Failed deleting tester of " + id.application(), e); + return Optional.of(error); + } } private Optional report(RunId id, DualLogger logger) { @@ -482,6 +495,7 @@ public class InternalStepRunner implements StepRunner { } catch (IllegalStateException e) { logger.log(INFO, "Job '" + id.type() + "'no longer supposed to run?:", e); + return Optional.of(error); } return Optional.of(running); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java index 0cb400004aa..f052c7d91ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Run.java @@ -14,6 +14,7 @@ import java.util.Optional; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.aborted; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.running; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.success; +import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; import static java.util.Objects.requireNonNull; @@ -173,15 +174,14 @@ public class Run { .iterator()); } - /** Returns the list of not-yet-succeeded run-always steps whose run-always prerequisites have all succeeded. */ + /** Returns the list of not-yet-run run-always steps whose run-always prerequisites have all run. */ private List forcedSteps() { return ImmutableList.copyOf(steps.entrySet().stream() - .filter(entry -> entry.getValue() != succeeded + .filter(entry -> entry.getValue() == unfinished && JobProfile.of(id.type()).alwaysRun().contains(entry.getKey()) && entry.getKey().prerequisites().stream() .filter(JobProfile.of(id.type()).alwaysRun()::contains) - .allMatch(step -> steps.get(step) == null - || steps.get(step) == succeeded)) + .allMatch(step -> steps.get(step) != unfinished)) .map(Map.Entry::getKey) .iterator()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java index 051a99074d1..a5f9ef86da4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/Step.java @@ -15,7 +15,7 @@ import java.util.List; * only the prerequisites of a step which are included in a run's profile will be considered. * Under normal circumstances, a step will run only after each of its prerequisites have succeeded. * When a run has failed, however, each of the always-run steps of the run's profile will be run, - * again in a topological order, and again requiring success of all their always-run prerequisites. + * again in a topological order, and requiring all their always-run prerequisites to have run. * * 2. A step will never run concurrently with its prerequisites. This is to ensure, e.g., that relevant * information from a failed run is stored, and that deployment does not occur after deactivation. -- cgit v1.2.3 From 672572db6b4dad8f9976b962e5525d1b68e7b16e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 10 May 2019 16:25:18 +0200 Subject: Avoid NPE when serialising unknown application versions --- .../controller/restapi/application/JobControllerApiHandlerHelper.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 48cf2a7824d..f3057baaf9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -351,6 +351,9 @@ class JobControllerApiHandlerHelper { private static void applicationVersionToSlime(Cursor versionObject, ApplicationVersion version) { versionObject.setString("hash", version.id()); + if (version.isUnknown()) + return; + versionObject.setLong("build", version.buildNumber().getAsLong()); Cursor sourceObject = versionObject.setObject("source"); sourceObject.setString("gitRepository", version.source().get().repository()); -- cgit v1.2.3 From 36586509a39dcacd030fead2ccd253fc53dfdb12 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 13 May 2019 09:36:43 +0200 Subject: Avoid NPE when serialising direct deployments --- .../restapi/application/JobControllerApiHandlerHelper.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index f3057baaf9a..d402f150725 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -327,9 +327,10 @@ class JobControllerApiHandlerHelper { /** Returns the status of the task represented by the given step, if it has started. */ private static Optional taskStatus(Step step, Run run) { - return run.readySteps().contains(step) ? Optional.of("running") - : run.steps().get(step) != unfinished ? Optional.of(run.steps().get(step).name()) - : Optional.empty(); + return run.readySteps().contains(step) ? Optional.of("running") + : Optional.ofNullable(run.steps().get(step)) + .filter(status -> status != unfinished) + .map(Step.Status::name); } /** Returns a response with the runs for the given job type. */ -- cgit v1.2.3 From 6e7b32b70ffd40bb98a3e483be3681294a8b996e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 13 May 2019 12:03:54 +0200 Subject: Check endpoints as part of install steps too --- .../controller/deployment/InternalStepRunner.java | 33 +++++++++++++++++++--- .../deployment/InternalDeploymentTester.java | 22 ++++++++------- .../deployment/InternalStepRunnerTest.java | 15 ++++++---- 3 files changed, 50 insertions(+), 20 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index fdadae065d2..401c8a5becb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -269,8 +269,14 @@ public class InternalStepRunner implements StepRunner { if ( nodesConverged(id.application(), id.type(), platform, logger) && servicesConverged(id.application(), id.type(), logger)) { - logger.log("Installation succeeded!"); - return Optional.of(running); + if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { + logger.log("Installation succeeded!"); + return Optional.of(running); + } + else if (timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); + return Optional.of(error); + } } if (timedOut(deployment.get(), installationTimeout)) { @@ -293,8 +299,14 @@ public class InternalStepRunner implements StepRunner { logger.log("Checking installation of tester container ..."); if ( nodesConverged(id.tester().id(), id.type(), platform, logger) && servicesConverged(id.tester().id(), id.type(), logger)) { - logger.log("Tester container successfully installed!"); - return Optional.of(running); + if (endpointsAvailable(id.tester().id(), id.type().zone(controller.system()), logger)) { + logger.log("Tester container successfully installed!"); + return Optional.of(running); + } + else if (timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Tester failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); + return Optional.of(error); + } } if (timedOut(deployment.get(), installationTimeout)) { @@ -306,6 +318,19 @@ public class InternalStepRunner implements StepRunner { return Optional.empty(); } + private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) { + logger.log("Attempting to find deployment endpoints ..."); + Map> endpoints = deploymentEndpoints(id, Set.of(zoneId)); + List messages = new ArrayList<>(); + messages.add("Found endpoints"); + endpoints.forEach((zone, uris) -> { + messages.add("- " + zone); + uris.forEach(uri -> messages.add(" |-- " + uri)); + }); + logger.log(messages); + return endpoints.containsKey(zoneId); + } + private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) { List nodes = controller.configServer().nodeRepository().list(type.zone(controller.system()), id, ImmutableSet.of(active, reserved)); List statuses = nodes.stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index 096a41e5b3f..20757820a4b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -182,6 +182,7 @@ public class InternalDeploymentTester { if (type == JobType.stagingTest) { // Do the initial deployment and installation of the real application. assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installInitialReal)); tester.configServer().convergeServices(appId, zone); + setEndpoints(appId, zone); run.versions().sourcePlatform().ifPresent(version -> tester.configServer().nodeRepository().doUpgrade(deployment, Optional.empty(), version)); runner.run(); assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installInitialReal)); @@ -193,6 +194,12 @@ public class InternalDeploymentTester { assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installReal)); tester.configServer().convergeServices(appId, zone); runner.run(); + if ( ! (run.versions().sourceApplication().isPresent() && type.isProduction()) + && type != JobType.stagingTest) { + assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installReal)); + setEndpoints(appId, zone); + } + runner.run(); assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installReal)); assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installTester)); @@ -201,17 +208,12 @@ public class InternalDeploymentTester { assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installTester)); tester.configServer().convergeServices(testerId.id(), zone); runner.run(); - assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installTester)); - - // All installation is complete. We now need endpoints, and the tests will then run, and cleanup finish. - assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.startTests)); + assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installTester)); setEndpoints(testerId.id(), zone); runner.run(); - if (!run.versions().sourceApplication().isPresent() || !type.isProduction()) { - assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.startTests)); - setEndpoints(appId, zone); - } - runner.run(); + assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installTester)); + + // All installation is complete and endpoints are ready, so tests may begin. assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.startTests)); assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.endTests)); @@ -222,7 +224,7 @@ public class InternalDeploymentTester { assertEquals(type.isProduction(), app().deployments().containsKey(zone)); assertTrue(tester.configServer().nodeRepository().list(zone, testerId.id()).isEmpty()); - if (!app().deployments().containsKey(zone)) + if ( ! app().deployments().containsKey(zone)) routing.removeEndpoints(deployment); routing.removeEndpoints(new DeploymentId(testerId.id(), zone)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index b53a7b39d61..02a4bfcdbe6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; +import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import org.junit.Before; @@ -156,6 +157,10 @@ public class InternalStepRunnerTest { public void waitsForEndpointsAndTimesOut() { tester.newRun(JobType.systemTest); + // Tester fails to show up for staging tests, and the real deployment for system tests. + tester.setEndpoints(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system())); + tester.setEndpoints(appId, JobType.stagingTest.zone(tester.tester().controller().system())); + tester.runner().run(); tester.configServer().convergeServices(appId, JobType.stagingTest.zone(tester.tester().controller().system())); tester.runner().run(); @@ -165,14 +170,10 @@ public class InternalStepRunnerTest { tester.configServer().convergeServices(testerId.id(), JobType.stagingTest.zone(tester.tester().controller().system())); tester.runner().run(); - // Tester fails to show up for system tests, and the real deployment for staging tests. - tester.setEndpoints(appId, JobType.systemTest.zone(tester.tester().controller().system())); - tester.setEndpoints(testerId.id(), JobType.stagingTest.zone(tester.tester().controller().system())); - tester.clock().advance(InternalStepRunner.endpointTimeout.plus(Duration.ofSeconds(1))); tester.runner().run(); - assertEquals(failed, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.startTests)); - assertEquals(failed, tester.jobs().last(appId, JobType.stagingTest).get().steps().get(Step.startTests)); + assertEquals(failed, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal)); + assertEquals(failed, tester.jobs().last(appId, JobType.stagingTest).get().steps().get(Step.installTester)); } @Test @@ -180,6 +181,7 @@ public class InternalStepRunnerTest { tester.newRun(JobType.systemTest); tester.runner().run(); tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system())); + tester.setEndpoints(appId, JobType.systemTest.zone(tester.tester().controller().system())); tester.runner().run(); assertEquals(succeeded, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal)); @@ -299,6 +301,7 @@ public class InternalStepRunnerTest { tester.runner().run(); // Job run order determined by JobType enum order per application. tester.configServer().convergeServices(appId, zone); + tester.setEndpoints(appId, zone); assertEquals(unfinished, tester.jobs().run(id).get().steps().get(Step.installReal)); assertEquals(otherPackage.hash(), tester.configServer().application(appId).get().applicationPackage().hash()); -- cgit v1.2.3 From f6937702737e9b8e0e92609ed5eaff21ffe36bad Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 13 May 2019 13:34:23 +0200 Subject: Return run number instead of undefined location URI --- .../controller/restapi/application/ApplicationApiHandler.java | 2 +- .../restapi/application/responses/deployment-job-accepted.json | 2 +- .../main/java/ai/vespa/hosted/api/ControllerHttpClient.java | 2 +- .../src/main/java/ai/vespa/hosted/api/DeploymentResult.java | 10 +++++----- 4 files changed, 8 insertions(+), 8 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index d32b3f009f4..27585d16be4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -919,7 +919,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Slime slime = new Slime(); Cursor rootObject = slime.setObject(); rootObject.setString("message", "Deployment started in " + runId); - rootObject.setString("location", controller.zoneRegistry().dashboardUrl(runId).toString()); + rootObject.setLong("run", runId.number()); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json index 66fce327701..ef0112f5ff8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json @@ -1,4 +1,4 @@ { "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1", - "location": "https://dashboard.tld/tenant1.application1/dev-us-east-1/1" + "run": 1 } \ No newline at end of file diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index eeeff2873f5..15cb9aa01ef 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -235,7 +235,7 @@ public abstract class ControllerHttpClient { private static DeploymentResult toDeploymentResult(HttpResponse response) { Inspector rootObject = toInspector(response); return new DeploymentResult(rootObject.field("message").asString(), - URI.create(rootObject.field("location").asString())); + rootObject.field("run").asLong()); } private static Slime toSlime(byte[] data) { diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java index f13453b7416..e78a7e926b4 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/DeploymentResult.java @@ -11,19 +11,19 @@ import java.net.URI; public class DeploymentResult { private final String message; - private final URI location; + private final long run; - public DeploymentResult(String message, URI location) { + public DeploymentResult(String message, long run) { this.message = message; - this.location = location; + this.run = run; } public String message() { return message; } - public URI location() { - return location; + public long run() { + return run; } } -- cgit v1.2.3 From fa6afcac80455a63d515d2aab1dea8a8f2086e9a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 14 May 2019 11:29:16 +0200 Subject: Better feedback when services are converged, or endpoints aren't ready --- .../vespa/hosted/controller/deployment/InternalStepRunner.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 401c8a5becb..5a301bdeeeb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -321,14 +321,18 @@ public class InternalStepRunner implements StepRunner { private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) { logger.log("Attempting to find deployment endpoints ..."); Map> endpoints = deploymentEndpoints(id, Set.of(zoneId)); + if ( ! endpoints.containsKey(zoneId)) { + logger.log("Endpoints not yet ready."); + return false; + } List messages = new ArrayList<>(); - messages.add("Found endpoints"); + messages.add("Found endpoints:"); endpoints.forEach((zone, uris) -> { messages.add("- " + zone); uris.forEach(uri -> messages.add(" |-- " + uri)); }); logger.log(messages); - return endpoints.containsKey(zoneId); + return true; } private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) { @@ -366,6 +370,8 @@ public class InternalStepRunner implements StepRunner { serviceStatus.currentGeneration() == -1 ? "not started!" : Long.toString(serviceStatus.currentGeneration()))) .collect(Collectors.toList()); logger.log(statuses); + if (statuses.isEmpty()) + logger.log("All services on wanted config generation."); return convergence.get().converged(); } -- cgit v1.2.3 From d894d32f14e90d3ad3b35700c667107df1cfca6c Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 14 May 2019 13:10:08 +0200 Subject: Display only endpoints for the relevant deployment in application/v4 --- .../com/yahoo/vespa/hosted/controller/ApplicationController.java | 7 +++++++ .../controller/restapi/application/ApplicationApiHandler.java | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index b88f1bbb8e1..81921641a25 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -708,6 +708,13 @@ public class ApplicationController { return curator.readRoutingPolicies(application); } + /** Returns all known routing policies for given deployment */ + public Set routingPolicies(DeploymentId deployment) { + return curator.readRoutingPolicies(deployment.applicationId()).stream() + .filter(policy -> policy.zone().equals(deployment.zoneId())) + .collect(Collectors.toUnmodifiableSet()); + } + /** Sort given list of applications by application ID */ private static List sort(List applications) { return applications.stream().sorted(Comparator.comparing(Application::id)).collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 27585d16be4..872f4d5d1af 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -585,7 +585,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add endpoint(s) defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies(deploymentId.applicationId())) { + for (var policy : controller.applications().routingPolicies(deploymentId)) { Cursor endpointObject = endpointArray.addObject(); Endpoint endpoint = policy.endpointIn(controller.system()); endpointObject.setString("cluster", policy.cluster().value()); -- cgit v1.2.3 From 29963c7368aa6d8bbc4cdbf682752dcef50495fe Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 14 May 2019 13:10:33 +0200 Subject: Check for roulting policies in InternalStepRunner --- .../controller/deployment/InternalStepRunner.java | 16 ++++++++--- .../controller/deployment/JobController.java | 9 ++++--- .../deployment/InternalStepRunnerTest.java | 31 +++++++++++++++++++++- 3 files changed, 48 insertions(+), 8 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 5a301bdeeeb..3233422ace6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -38,6 +38,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; +import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; @@ -626,10 +627,17 @@ public class InternalStepRunner implements StepRunner { /** Returns all endpoints for all current deployments of the given real application. */ private Map> deploymentEndpoints(ApplicationId id, Iterable zones) { ImmutableMap.Builder> deployments = ImmutableMap.builder(); - for (ZoneId zone : zones) - controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) - .filter(endpoints -> ! endpoints.isEmpty()) - .ifPresent(endpoints -> deployments.put(zone, endpoints)); + for (ZoneId zone : zones) { + Set policies = controller.applications().routingPolicies(new DeploymentId(id, zone)); + if ( ! policies.isEmpty()) + deployments.put(zone, policies.stream() + .map(policy -> policy.endpointIn(controller.system()).url()) + .collect(Collectors.toUnmodifiableList())); + else + controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) + .filter(endpoints -> ! endpoints.isEmpty()) + .ifPresent(endpoints -> deployments.put(zone, endpoints)); + } return deployments.build(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 245af60d0ad..0def14ed227 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -399,9 +399,12 @@ public class JobController { /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */ Optional testerEndpoint(RunId id) { - ApplicationId tester = id.tester().id(); - return controller.applications().getDeploymentEndpoints(new DeploymentId(tester, id.type().zone(controller.system()))) - .flatMap(uris -> uris.stream().findAny()); + DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); + return controller.applications().routingPolicies(testerId).stream() + .findAny() + .map(policy -> policy.endpointIn(controller.system()).url()) + .or(() -> controller.applications().getDeploymentEndpoints(testerId) + .flatMap(uris -> uris.stream().findAny())); } // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 02a4bfcdbe6..40f59dc146f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; @@ -18,9 +19,9 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import org.junit.Before; import org.junit.Test; @@ -34,6 +35,7 @@ import java.time.Duration; import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.Future; @@ -47,6 +49,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTes import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; +import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -205,6 +208,32 @@ public class InternalStepRunnerTest { assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.startTests)); } + @Test + public void alternativeEndpointsAreDetected() { + tester.newRun(JobType.systemTest); + tester.runner().run();; + tester.configServer().convergeServices(appId, JobType.systemTest.zone(tester.tester().controller().system())); + tester.configServer().convergeServices(testerId.id(), JobType.systemTest.zone(tester.tester().controller().system())); + assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal)); + assertEquals(unfinished, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester)); + + tester.tester().controller().curator().writeRoutingPolicies(appId, Set.of(new RoutingPolicy(appId, + ClusterSpec.Id.from("default"), + JobType.systemTest.zone(tester.tester().controller().system()), + HostName.from("host"), + Optional.empty(), + emptySet()))); + tester.tester().controller().curator().writeRoutingPolicies(testerId.id(), Set.of(new RoutingPolicy(testerId.id(), + ClusterSpec.Id.from("default"), + JobType.systemTest.zone(tester.tester().controller().system()), + HostName.from("host"), + Optional.empty(), + emptySet()))); + tester.runner().run();; + assertEquals(succeeded, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installReal)); + assertEquals(succeeded, tester.jobs().last(appId, JobType.systemTest).get().steps().get(Step.installTester)); + } + @Test public void testsFailIfTesterRestarts() { RunId id = tester.startSystemTestTests(); -- cgit v1.2.3 From 45e950796cf6b74921bad6445d6e84650a3e5937 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 14 May 2019 13:53:40 +0200 Subject: Always use value for SystemName, Environment and RegionName --- .../main/java/com/yahoo/vespa/model/container/ContainerCluster.java | 2 +- .../vespa/config/server/maintenance/ConfigServerMaintenance.java | 2 +- .../com/yahoo/vespa/hosted/controller/application/Endpoint.java | 2 +- .../vespa/hosted/controller/deployment/InternalStepRunner.java | 2 +- .../yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java | 2 +- .../vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java | 6 +++--- .../vespa/hosted/controller/deployment/InternalStepRunnerTest.java | 3 +-- .../vespa/hosted/controller/restapi/ControllerContainerTest.java | 2 +- 8 files changed, 10 insertions(+), 11 deletions(-) (limited to 'controller-server') diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java index 1f0c3d6f84e..e002e722a5c 100755 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java @@ -609,7 +609,7 @@ public abstract class ContainerCluster */ @Override public void getConfig(ConfigserverConfig.Builder builder) { - builder.system(zone.system().name()); + builder.system(zone.system().value()); builder.environment(zone.environment().value()); builder.region(zone.region().value()); } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java index 505ae38161d..e2495de18ba 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/maintenance/ConfigServerMaintenance.java @@ -53,7 +53,7 @@ public class ConfigServerMaintenance extends AbstractComponent { DefaultTimes(ConfigserverConfig configserverConfig) { this.defaultInterval = Duration.ofMinutes(configserverConfig.maintainerIntervalMinutes()); - boolean isCd = configserverConfig.system().equals(SystemName.cd.name()); + boolean isCd = configserverConfig.system().equals(SystemName.cd.value()); // TODO: Want job control or feature flag to control when to run this, for now use a very // long interval to avoid running the maintainer except in CD this.tenantsMaintainerInterval = isCd diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index ce7af03aa7e..48ae8417e6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -141,7 +141,7 @@ public class Endpoint { private static String systemPart(SystemName system, String separator) { if (system == SystemName.main || isPublic(system)) return ""; - return system.name() + separator; + return system.value() + separator; } private static String dnsSuffix(SystemName system, boolean legacy) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 401c8a5becb..fa653997da4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -711,7 +711,7 @@ public class InternalStepRunner implements StepRunner { root.setString("application", id.serializedForm()); root.setString("zone", testerZone.value()); - root.setString("system", system.name()); + root.setString("system", system.value()); Cursor endpointsObject = root.setObject("endpoints"); deployments.forEach((zone, endpoints) -> { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 449b8c51acd..5e3f21c6b98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -212,7 +212,7 @@ public class MetricsReporter extends Maintainer { } private static void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) { - nodeResult.rows.removeIf(node -> !system.name().equals(node.getValue("system").orElse("main"))); + nodeResult.rows.removeIf(node -> !system.value().equals(node.getValue("system").orElse("main"))); } private static Map dimensions(ApplicationId application) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index 01d9a01a316..a208249b410 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -5,14 +5,14 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.jdisc.http.HttpRequest.Method; import com.yahoo.log.LogLevel; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.identity.ServiceIdentityProvider; import com.yahoo.vespa.athenz.tls.AthenzIdentityVerifier; import com.yahoo.vespa.athenz.utils.AthenzIdentities; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import org.apache.http.Header; import org.apache.http.client.config.RequestConfig; @@ -116,7 +116,7 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { for (ZoneId zoneId : zones.ids()) { responseStructure.uris.add(proxyRequest.getScheme() + "://" + proxyRequest.getControllerPrefix() + - zoneId.environment().name() + "/" + zoneId.region().value()); + zoneId.environment().value() + "/" + zoneId.region().value()); } JsonNode node = mapper.valueToTree(responseStructure); return new ProxyResponse(proxyRequest, node.toString(), 200, Optional.empty(), "application/json"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 02a4bfcdbe6..c97c29e2441 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; -import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import org.junit.Before; @@ -251,7 +250,7 @@ public class InternalStepRunnerTest { Inspector configObject = SlimeUtils.jsonToSlime(tester.cloud().config()).get(); assertEquals(appId.serializedForm(), configObject.field("application").asString()); assertEquals(JobType.systemTest.zone(tester.tester().controller().system()).value(), configObject.field("zone").asString()); - assertEquals(tester.tester().controller().system().name(), configObject.field("system").asString()); + assertEquals(tester.tester().controller().system().value(), configObject.field("system").asString()); assertEquals(1, configObject.field("endpoints").children()); assertEquals(1, configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).entries()); configObject.field("endpoints").field(JobType.systemTest.zone(tester.tester().controller().system()).value()).traverse((ArrayTraverser) (__, endpoint) -> diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index b81edaccd09..8e12268c197 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -48,7 +48,7 @@ public class ControllerContainerTest { " 10\n" + " \n" + " \n" + - " " + system().name() + "\n" + + " " + system().value() + "\n" + " \n" + " \n" + " \n" + -- cgit v1.2.3 From 4b237f6b29874b74cd697a5827077977b2dd1239 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 14 May 2019 14:31:37 +0200 Subject: Do not use valueOf() --- .../vespa/hosted/controller/integration/ZoneRegistryMock.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 4248a513950..12b0f2a930a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -10,13 +10,13 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; -import com.yahoo.vespa.athenz.api.AthenzService; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneFilterMock; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.athenz.api.AthenzService; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry; import java.net.URI; @@ -42,7 +42,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Inject public ZoneRegistryMock(ConfigserverConfig config) { - this(SystemName.valueOf(config.system())); + this(SystemName.from(config.system())); } public ZoneRegistryMock() { -- cgit v1.2.3 From 36fdf72e6d5529fb4aa20a9eeaf08b15f8131846 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 14 May 2019 15:13:47 +0200 Subject: Add public cd DNS suffix --- .../java/com/yahoo/vespa/hosted/controller/application/Endpoint.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 48ae8417e6c..1b90a43421b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -21,6 +21,7 @@ public class Endpoint { public static final String YAHOO_DNS_SUFFIX = ".vespa.yahooapis.com"; public static final String OATH_DNS_SUFFIX = ".vespa.oath.cloud"; public static final String PUBLIC_DNS_SUFFIX = ".public.vespa.oath.cloud"; + public static final String PUBLIC_CD_DNS_SUFFIX = ".public-cd.vespa.oath.cloud"; private final URI url; private final Scope scope; @@ -153,6 +154,8 @@ public class Endpoint { case Public: case vaas: return PUBLIC_DNS_SUFFIX; + case PublicCd: + return PUBLIC_CD_DNS_SUFFIX; default: throw new IllegalArgumentException("No DNS suffix declared for system " + system); } } -- cgit v1.2.3 From 0b6387d64f827175b04d0d26fa4eef4223cadca9 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 14 May 2019 21:59:28 +0200 Subject: Unit test run response --- .../deployment/InternalDeploymentTester.java | 5 +++++ .../JobControllerApiHandlerHelperTest.java | 6 ++++++ .../responses/dev-aws-us-east-1a-runs.json | 21 +++++++++++++++++++++ 3 files changed, 32 insertions(+) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java index 20757820a4b..a992ce1e3de 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalDeploymentTester.java @@ -200,6 +200,11 @@ public class InternalDeploymentTester { setEndpoints(appId, zone); } runner.run(); + if (type.environment().isManuallyDeployed()) { + assertEquals(Step.Status.succeeded, jobs.run(run.id()).get().steps().get(Step.installReal)); + assertTrue(jobs.run(run.id()).get().hasEnded()); + return; + } assertEquals(Step.Status.succeeded, jobs.active(run.id()).get().steps().get(Step.installReal)); assertEquals(unfinished, jobs.active(run.id()).get().steps().get(Step.installTester)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 212ee272a63..bd6695f9444 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -6,6 +6,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import org.json.JSONException; import org.json.JSONObject; @@ -19,6 +20,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; +import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; @@ -28,6 +30,7 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud.Status.FAILURE; import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId; +import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.testerId; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.deploymentFailed; import static com.yahoo.vespa.hosted.controller.deployment.RunStatus.installationFailed; @@ -111,6 +114,9 @@ public class JobControllerApiHandlerHelperTest { assertResponse(JobControllerApiHandlerHelper.runDetailsResponse(tester.jobs(), tester.jobs().last(appId, productionUsEast3).get().id(), "0"), "us-east-3-log-without-first.json"); assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.tester().controller(), appId, URI.create("https://some.url:43/root/")), "overview.json"); + tester.jobs().deploy(appId, JobType.devAwsUsEast2a, Optional.empty(), applicationPackage); + tester.runJob(JobType.devAwsUsEast2a); + assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(appId, stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json"); } private void compare(HttpResponse response, String expected) throws JSONException, IOException { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json new file mode 100644 index 00000000000..a677eba6778 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json @@ -0,0 +1,21 @@ +{ + "1": { + "id": 1, + "status": "success", + "start": 0, + "end": 0, + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, + "log": "https://some.url:43/root/run/1" + } +} -- cgit v1.2.3 From 54609d69fcfbb790bced584a32ceee91be69e446 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 15 May 2019 14:20:23 +0200 Subject: Actually test the response >_< --- .../JobControllerApiHandlerHelperTest.java | 3 ++- .../responses/dev-aws-us-east-1a-runs.json | 21 --------------------- .../responses/dev-aws-us-east-2a-runs.json | 21 +++++++++++++++++++++ 3 files changed, 23 insertions(+), 22 deletions(-) delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index bd6695f9444..8580677431c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -23,6 +23,7 @@ import java.time.Instant; import java.util.Optional; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.devAwsUsEast2a; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsCentral1; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsEast3; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; @@ -116,7 +117,7 @@ public class JobControllerApiHandlerHelperTest { tester.jobs().deploy(appId, JobType.devAwsUsEast2a, Optional.empty(), applicationPackage); tester.runJob(JobType.devAwsUsEast2a); - assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(appId, stagingTest), URI.create("https://some.url:43/root")), "staging-runs.json"); + assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(appId, devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); } private void compare(HttpResponse response, String expected) throws JSONException, IOException { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json deleted file mode 100644 index a677eba6778..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-1a-runs.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "1": { - "id": 1, - "status": "success", - "start": 0, - "end": 0, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "unknown" - }, - "steps": { - "deployReal": "succeeded", - "installReal": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded" - }, - "log": "https://some.url:43/root/run/1" - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json new file mode 100644 index 00000000000..82bdc4ca195 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-aws-us-east-2a-runs.json @@ -0,0 +1,21 @@ +{ + "1": { + "id": 1, + "status": "success", + "start": 102000, + "end": 102000, + "wantedPlatform": "7.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, + "log": "https://some.url:43/root/run/1" + } +} -- cgit v1.2.3 From f896f2480b55014b1632801bfe2efbea6e5afa87 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 15 May 2019 14:30:16 +0200 Subject: Use routing layer by default, and policy as fallback, since latter does not yet work --- .../hosted/controller/deployment/InternalStepRunner.java | 16 +++++++--------- .../hosted/controller/deployment/JobController.java | 10 +++++----- 2 files changed, 12 insertions(+), 14 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index f7f7dc2d800..5db207eca55 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -628,15 +628,13 @@ public class InternalStepRunner implements StepRunner { private Map> deploymentEndpoints(ApplicationId id, Iterable zones) { ImmutableMap.Builder> deployments = ImmutableMap.builder(); for (ZoneId zone : zones) { - Set policies = controller.applications().routingPolicies(new DeploymentId(id, zone)); - if ( ! policies.isEmpty()) - deployments.put(zone, policies.stream() - .map(policy -> policy.endpointIn(controller.system()).url()) - .collect(Collectors.toUnmodifiableList())); - else - controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) - .filter(endpoints -> ! endpoints.isEmpty()) - .ifPresent(endpoints -> deployments.put(zone, endpoints)); + controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) + .filter(endpoints -> ! endpoints.isEmpty()) + .or(() -> Optional.of(controller.applications().routingPolicies(new DeploymentId(id, zone)).stream() + .map(policy -> policy.endpointIn(controller.system()).url()) + .collect(Collectors.toUnmodifiableList())) + .filter(endpoints -> ! endpoints.isEmpty())) + .ifPresent(endpoints -> deployments.put(zone, endpoints)); } return deployments.build(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 0def14ed227..3ba12e03d3e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -400,11 +400,11 @@ public class JobController { /** Returns a URI of the tester endpoint retrieved from the routing generator, provided it matches an expected form. */ Optional testerEndpoint(RunId id) { DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); - return controller.applications().routingPolicies(testerId).stream() - .findAny() - .map(policy -> policy.endpointIn(controller.system()).url()) - .or(() -> controller.applications().getDeploymentEndpoints(testerId) - .flatMap(uris -> uris.stream().findAny())); + return controller.applications().getDeploymentEndpoints(testerId) + .flatMap(uris -> uris.stream().findAny()) + .or(() -> controller.applications().routingPolicies(testerId).stream() + .findAny() + .map(policy -> policy.endpointIn(controller.system()).url())); } // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. -- cgit v1.2.3 From da186e9856eab07fe1b3a7f63009fe6e741e3091 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 16 May 2019 14:52:37 +0200 Subject: Use the name "tester" for the tester container cluster --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 2 +- controller-server/src/test/resources/test_runner_services.xml-cd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 5db207eca55..fe45840bee2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -659,7 +659,7 @@ public class InternalStepRunner implements StepRunner { String servicesXml = "\n" + "\n" + - " \n" + + " \n" + "\n" + " \n" + " \n" + diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index 15d3c98b995..6a01b612c3c 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -1,6 +1,6 @@ - + -- cgit v1.2.3 From c45ec4e70b797b3a1f2ca7448a047342628331ec Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 20 May 2019 11:21:14 +0200 Subject: Display which deployments hinder application deletion --- .../java/com/yahoo/vespa/hosted/controller/ApplicationController.java | 4 +++- .../restapi/application/responses/delete-with-active-deployments.json | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 81921641a25..9a489f8379d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -556,7 +556,9 @@ public class ApplicationController { // TODO: Make this one transaction when database is moved to ZooKeeper instances.forEach(id -> lockOrThrow(id, application -> { if ( ! application.get().deployments().isEmpty()) - throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments"); + throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments in: " + + application.get().deployments().keySet().stream().map(ZoneId::toString) + .sorted().collect(Collectors.joining(", "))); curator.removeApplication(id); applicationStore.removeAll(id); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json index f85ac6dbf8b..2b17d55627a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Could not delete 'application 'tenant1.application1'': It has active deployments" + "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-west-1 in default, zone prod.us-central-1 in default" } -- cgit v1.2.3 From 348fdb45731b12aa9f076751d7beff6ba07459ce Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 20 May 2019 11:22:02 +0200 Subject: Kick job runner when deploying, to reduce latency and improve UX --- .../yahoo/vespa/hosted/controller/deployment/JobController.java | 8 ++++++++ .../com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java | 1 + .../hosted/controller/restapi/application/ApplicationApiTest.java | 5 +++++ 3 files changed, 14 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 3ba12e03d3e..5241267460c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -22,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.JobStatus; +import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -72,6 +73,8 @@ public class JobController { private final TesterCloud cloud; private final Badges badges; + private AtomicReference> runner = new AtomicReference<>(__ -> { }); + public JobController(Controller controller, RunDataStore runDataStore, TesterCloud testerCloud) { this.controller = controller; this.curator = controller.curator(); @@ -82,6 +85,7 @@ public class JobController { public TesterCloud cloud() { return cloud; } public int historyLength() { return historyLength; } + public void setRunner(Consumer runner) { this.runner.set(runner); } /** Rewrite all job data with the newest format. */ public void updateStorage() { @@ -317,12 +321,16 @@ public class JobController { ApplicationVersion.unknown, Optional.empty(), Optional.empty())); + + runner.get().accept(last(id, type).get()); }); } /** Aborts a run and waits for it complete. */ private void abortAndWait(RunId id) { abort(id); + runner.get().accept(last(id.application(), id.type()).get()); + while ( ! last(id.application(), id.type()).get().hasEnded()) { try { Thread.sleep(100); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index 47e17b34d2a..fb1e0599a4c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -43,6 +43,7 @@ public class JobRunner extends Maintainer { public JobRunner(Controller controller, Duration duration, JobControl jobControl, ExecutorService executors, StepRunner runner) { super(controller, duration, jobControl); this.jobs = controller.jobController(); + this.jobs.setRunner(this::advance); this.executors = executors; this.runner = runner; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index aeb9d02336a..4ec075c3c16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -658,6 +658,11 @@ public class ApplicationApiTest extends ControllerContainerTest { .screwdriverIdentity(SCREWDRIVER_ID), "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); + // DELETE the deployment to dev + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default", DELETE) + .userIdentity(USER_ID), + "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default"); + // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), -- cgit v1.2.3 From 588c95846e0b0a7a0cf69d1466a96b94c49841c0 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 20 May 2019 13:00:19 +0200 Subject: Cleanup --- .../maintenance/ClusterInfoMaintainer.java | 29 ++++++---------------- 1 file changed, 8 insertions(+), 21 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index c9b07ada854..496e09954e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -8,11 +8,9 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; -import java.io.IOException; import java.time.Duration; import java.util.HashMap; import java.util.List; @@ -22,8 +20,8 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Maintain info about hardware, hostnames and cluster specifications. - *

+ * Maintains information about hardware, hostnames and cluster specifications. + * * This is used to calculate cost metrics for the application api. * * @author smorgrav @@ -42,17 +40,17 @@ public class ClusterInfoMaintainer extends Maintainer { this.nodeRepositoryClient = nodeRepositoryClient; } - private static String clusterid(NodeRepositoryNode node) { + private static String clusterId(NodeRepositoryNode node) { return node.getMembership().clusterid; } - private Map getClusterInfo(NodeList nodes, ZoneId zone) { + private Map getClusterInfo(NodeList nodes) { Map infoMap = new HashMap<>(); // Group nodes by clusterid Map> clusters = nodes.nodes().stream() .filter(node -> node.getMembership() != null) - .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterid)); + .collect(Collectors.groupingBy(ClusterInfoMaintainer::clusterId)); // For each cluster - get info for (String id : clusters.keySet()) { @@ -65,16 +63,6 @@ public class ClusterInfoMaintainer extends Maintainer { double cpu = 0; double mem = 0; double disk = 0; - // TODO: This code was never run. Reenable when flavours are available from a FlavorRegistry or something, or remove. - /*if (zone.nodeFlavors().isPresent()) { - Optional flavorOptional = zone.nodeFlavors().get().getFlavor(node.flavor); - if ((flavorOptional.isPresent())) { - Flavor flavor = flavorOptional.get(); - cpu = flavor.getMinCpuCores(); - mem = flavor.getMinMainMemoryAvailableGb(); - disk = flavor.getMinMainMemoryAvailableGb(); - } - }*/ // Add to map List hostnames = clusterNodes.stream().map(NodeRepositoryNode::getHostname).collect(Collectors.toList()); @@ -96,12 +84,11 @@ public class ClusterInfoMaintainer extends Maintainer { deploymentId.applicationId().tenant().value(), deploymentId.applicationId().application().value(), deploymentId.applicationId().instance().value()); - Map clusterInfo = getClusterInfo(nodes, deployment.zone()); + Map clusterInfo = getClusterInfo(nodes); controller().applications().lockIfPresent(application.id(), lockedApplication -> controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo))); - } - catch (IOException | IllegalArgumentException e) { - log.log(Level.WARNING, "Failing getting cluster info of for " + deploymentId, e); + } catch (Exception e) { + log.log(Level.WARNING, "Failing getting cluster information for " + deploymentId, e); } } } -- cgit v1.2.3 From 1de39885e3a4a4a2c3147ea5ddf4f1733048294e Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 20 May 2019 13:00:28 +0200 Subject: Fix ClusterInfoMaintainer Cost is not declared for all active flavors. Possibly due to the introduction of user-defined flavors in #9291. --- .../vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index 496e09954e5..c99e4f5951b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -66,7 +66,8 @@ public class ClusterInfoMaintainer extends Maintainer { // Add to map List hostnames = clusterNodes.stream().map(NodeRepositoryNode::getHostname).collect(Collectors.toList()); - ClusterInfo inf = new ClusterInfo(node.getFlavor(), node.getCost(), cpu, mem, disk, + int cost = node.getCost() == null ? 0 : node.getCost(); // Cost is not guaranteed to be defined for all flavors + ClusterInfo inf = new ClusterInfo(node.getFlavor(), cost, cpu, mem, disk, ClusterSpec.Type.from(node.getMembership().clustertype), hostnames); infoMap.put(new ClusterSpec.Id(id), inf); } -- cgit v1.2.3 From c979d36586745ca159d50ffc87f366efb3d6d093 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Mon, 20 May 2019 13:14:28 +0200 Subject: Reduce DeploymentTrigger log level --- .../yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index a8130d60cc5..d45d5b586fa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -92,7 +92,7 @@ public class DeploymentTrigger { * trigger next. */ public void notifyOfCompletion(JobReport report) { - log.log(LogLevel.INFO, String.format("Notified of %s for %s of %s (%d)", + log.log(LogLevel.DEBUG, String.format("Notified of %s for %s of %s (%d)", report.jobError().map(e -> e.toString() + " error") .orElse("success"), report.jobType(), @@ -181,7 +181,7 @@ public class DeploymentTrigger { * the project id is removed from the application owning the job, to prevent further trigger attempts. */ public boolean trigger(Job job) { - log.log(LogLevel.INFO, String.format("Triggering %s: %s", job, job.triggering)); + log.log(LogLevel.DEBUG, String.format("Triggering %s: %s", job, job.triggering)); try { applications().lockOrThrow(job.applicationId(), application -> { if (application.get().deploymentJobs().deployedInternally()) -- cgit v1.2.3 From 96c17d9d72183e604e50e6a408fd52aa762da049 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 20 May 2019 15:24:23 +0200 Subject: Allocate 50% of remaining node memory to surefire tests --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 2 +- controller-server/src/test/resources/test_runner_services.xml-cd | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index fe45840bee2..78e1d4756fb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -654,7 +654,7 @@ public class InternalStepRunner implements StepRunner { String flavor = testerFlavor.orElse("d-1-4-50"); int memoryGb = Integer.parseInt(flavor.split("-")[2]); // Memory available in tester container. int jdiscMemoryPercentage = (int) Math.ceil(200.0 / memoryGb); // 2Gb memory for tester application (excessive?). - int testMemoryMb = 768 * (memoryGb - 2); // Memory allocated to Surefire running tests. ≥25% left for other stuff. + int testMemoryMb = 512 * (memoryGb - 2); // Memory allocated to Surefire running tests. ≥25% left for other stuff. String servicesXml = "\n" + diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index 6a01b612c3c..df276f9bbbc 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -5,7 +5,7 @@ artifacts - 7680 + 5120 -- cgit v1.2.3 From cf4622727778158a2e18e9756bea57b8dc5a1b1a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 21 May 2019 17:43:56 +0200 Subject: Fix some endpoint visibility issues --- .../controller/deployment/InternalStepRunner.java | 32 +++++++++------------- 1 file changed, 13 insertions(+), 19 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 78e1d4756fb..6370dd6519c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -388,41 +388,35 @@ public class InternalStepRunner implements StepRunner { logger.log("Attempting to find endpoints ..."); Map> endpoints = deploymentEndpoints(id.application(), zones); + if ( ! endpoints.containsKey(id.type().zone(controller.system())) && timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); + return Optional.of(error); + } List messages = new ArrayList<>(); - messages.add("Found endpoints"); + messages.add("Found endpoints:"); endpoints.forEach((zone, uris) -> { messages.add("- " + zone); uris.forEach(uri -> messages.add(" |-- " + uri)); }); logger.log(messages); - if ( ! endpoints.containsKey(id.type().zone(controller.system()))) { - if (timedOut(deployment.get(), endpointTimeout)) { - logger.log(WARNING, "Endpoints failed to show up within " + endpointTimeout.toMinutes() + " minutes!"); - return Optional.of(error); - } - logger.log("Endpoints for the deployment to test are not yet ready."); - return Optional.empty(); + Optional testerEndpoint = controller.jobController().testerEndpoint(id); + if (testerEndpoint.isEmpty() && timedOut(deployment.get(), endpointTimeout)) { + logger.log(WARNING, "Endpoints for the tester container vanished again, while it was still active!"); + return Optional.of(error); } - Map> clusters = listClusters(id.application(), zones); - - Optional testerEndpoint = controller.jobController().testerEndpoint(id); - if (testerEndpoint.isPresent() && controller.jobController().cloud().ready(testerEndpoint.get())) { + if (controller.jobController().cloud().ready(testerEndpoint.get())) { logger.log("Starting tests ..."); controller.jobController().cloud().startTests(testerEndpoint.get(), TesterCloud.Suite.of(id.type()), testConfig(id.application(), id.type().zone(controller.system()), - controller.system(), endpoints, clusters)); + controller.system(), endpoints, + listClusters(id.application(), zones))); return Optional.of(running); } - if (timedOut(deployment.get(), endpointTimeout)) { - logger.log(WARNING, "Endpoint for tester failed to show up within " + endpointTimeout.toMinutes() + " minutes of real deployment!"); - return Optional.of(error); - } - - logger.log("Endpoints of tester container not yet available."); + logger.log("Tester container not yet ready."); return Optional.empty(); } -- cgit v1.2.3 From bbb2a4d2e6bdeff2fa86c08e0f5434816e1b1037 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 22 May 2019 09:47:48 +0200 Subject: Upgrade config server together with config server host --- .../controller/application/SystemApplication.java | 2 +- .../controller/maintenance/SystemUpgraderTest.java | 17 ++++++----------- 2 files changed, 7 insertions(+), 12 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 708133d2a07..26233c998f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -22,7 +22,7 @@ public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), - configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config, configServerHost), + configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), zone(ApplicationId.from("hosted-vespa", "routing", "default"), Set.of(NodeType.proxy, NodeType.host), configServerHost, proxyHost, configServer); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 7f558131dd0..d28b36265e3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -196,11 +196,10 @@ public class SystemUpgraderTest { // System upgrades in zone 1: systemUpgrader.maintain(); - List allExceptZoneAndConfig = List.of(SystemApplication.configServerHost, - SystemApplication.proxyHost); - completeUpgrade(allExceptZoneAndConfig, version2, zone1); - systemUpgrader.maintain(); - completeUpgrade(SystemApplication.configServer, version2, zone1); + List allExceptZone = List.of(SystemApplication.configServerHost, + SystemApplication.configServer, + SystemApplication.proxyHost); + completeUpgrade(allExceptZone, version2, zone1); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone1); convergeServices(SystemApplication.zone, zone1); @@ -208,9 +207,7 @@ public class SystemUpgraderTest { // zone 2 and 3: systemUpgrader.maintain(); - completeUpgrade(allExceptZoneAndConfig, version2, zone2, zone3); - systemUpgrader.maintain(); - completeUpgrade(SystemApplication.configServer, version2, zone2, zone3); + completeUpgrade(allExceptZone, version2, zone2, zone3); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone2, zone3); convergeServices(SystemApplication.zone, zone2, zone3); @@ -218,9 +215,7 @@ public class SystemUpgraderTest { // zone 4: systemUpgrader.maintain(); - completeUpgrade(allExceptZoneAndConfig, version2, zone4); - systemUpgrader.maintain(); - completeUpgrade(SystemApplication.configServer, version2, zone4); + completeUpgrade(allExceptZone, version2, zone4); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone4); -- cgit v1.2.3 From 98d07eba9fcf4ccb2bc8c4762444c3872f5cb3ce Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 22 May 2019 12:49:31 +0200 Subject: Add a memory fence for application deployment reads in step runner --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 6370dd6519c..83a347819f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -456,7 +456,7 @@ public class InternalStepRunner implements StepRunner { private Optional copyVespaLogs(RunId id, DualLogger logger) { ZoneId zone = id.type().zone(controller.system()); - if (controller.applications().require(id.application()).deployments().containsKey(zone)) + if (deployment(id.application(), id.type()).isPresent()) try { logger.log("Copying Vespa log from nodes of " + id.application() + " in " + zone + " ..."); List entries = new ArrayList<>(); @@ -566,6 +566,7 @@ public class InternalStepRunner implements StepRunner { /** Returns the real application with the given id. */ private Application application(ApplicationId id) { + controller.applications().lockOrThrow(id, __ -> { }); // Memory fence. return controller.applications().require(id); } -- cgit v1.2.3 From 17a40660f334a3eb1543b90143c574713f3b0a68 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 22 May 2019 14:27:59 +0200 Subject: Disable TLSv1.3 in manually configured https endpoint --- .../ConfigserverSslContextFactoryProvider.java | 4 ++++ .../hosted/controller/tls/ControllerSslContextFactoryProvider.java | 3 +++ 2 files changed, 7 insertions(+) (limited to 'controller-server') diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 61ac8f7a7e2..bb3216ba3ba 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -109,6 +109,9 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp AthenzService configserverIdentity, ZtsClient ztsClient, AthenzProviderServiceConfig.Zones zoneConfig) { + + // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints) + SslContextFactory factory = new SslContextFactory(); factory.setWantClientAuth(true); @@ -124,6 +127,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp .orElseGet(() -> updateKeystore(configserverIdentity, generateKeystorePassword(), keyProvider, ztsClient, zoneConfig)); factory.setKeyStore(keyStore); factory.setKeyStorePassword(""); + factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) return factory; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index dcc61b13bab..84d43314412 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -56,6 +56,8 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple /** Create a SslContextFactory backed by an in-memory key and trust store */ private SslContextFactory createSslContextFactory(int port) { + // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints). + SslContextFactory factory = new SslContextFactory(); if (port != 443) { factory.setWantClientAuth(true); @@ -63,6 +65,7 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple factory.setTrustStore(truststore); factory.setKeyStore(keystore); factory.setKeyStorePassword(""); + factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) return factory; } -- cgit v1.2.3 From eae2b9349be3721f21dfb5e593f9f2a5fe93e948 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 23 May 2019 09:03:17 +0200 Subject: Simplify --- .../vespa/hosted/controller/versions/OsVersionStatus.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index a18c1f47036..d55855a2f36 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -74,13 +74,11 @@ public class OsVersionStatus { controller.configServer().nodeRepository().list(zone, application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region())) - .forEach(node -> versions.compute(new OsVersion(node.version(), zone.cloud()), (ignored, nodes) -> { - if (nodes == null) { - nodes = new ArrayList<>(); - } - nodes.add(node); - return nodes; - })); + .forEach(node -> { + var version = new OsVersion(node.version(), zone.cloud()); + versions.putIfAbsent(version, new ArrayList<>()); + versions.get(version).add(node); + }); } } -- cgit v1.2.3 From 6a1028b321791e48060a5305d855475caa2b0b58 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 22 May 2019 15:42:56 +0200 Subject: Refresh routing policies on deploy (de)activation Refreshing routing policies and performing the necessary DNS updates are somewhat time sensitive, especially in manually deployed environments, hence it makes sense that this should be done as early as possible. After introducing queuing of name service requests in #9224 it became obvious that the asynchronous behaviour of `RoutingPolicyMaintainer` is no longer needed. Because name service requests are now executed asynchronously by default, we can refresh policies during deployment (de)activation without worrying about DNS service failures or rate limits. Benefits of this change: - Reduces worst-case DNS propagation time by 5 minutes. - We no longer need to update *all* routing policies in the system when refreshing policies. This both reduces number of queued name service requests and distributes them over a longer duration. - Implementation is simplified since the system-wide dimension disappears. - Fetching of load balancers from config servers conincides with deployment and are thus spread over a longer duration. --- .../api/integration/configserver/ConfigServer.java | 6 +- .../hosted/controller/ApplicationController.java | 15 +- .../maintenance/ControllerMaintenance.java | 3 - .../controller/maintenance/RoutingPolicies.java | 187 +++++++++++++++ .../maintenance/RoutingPolicyMaintainer.java | 224 ------------------ .../vespa/hosted/controller/ControllerTest.java | 2 +- .../deployment/ApplicationPackageBuilder.java | 5 + .../controller/deployment/DeploymentTester.java | 6 +- .../controller/integration/ConfigServerMock.java | 16 +- .../controller/maintenance/DnsMaintainerTest.java | 4 +- .../maintenance/MetricsReporterTest.java | 2 +- .../maintenance/RoutingPoliciesTest.java | 237 +++++++++++++++++++ .../maintenance/RoutingPolicyMaintainerTest.java | 250 --------------------- .../restapi/controller/responses/maintenance.json | 3 - 14 files changed, 462 insertions(+), 498 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index bcad82a23c1..43ae29ee922 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,11 +1,12 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.IOException; @@ -76,4 +77,7 @@ public interface ConfigServer { /** Get all load balancers in given zone */ List getLoadBalancers(ZoneId zone); + /** Get all load balancers for application in given zone */ + List getLoadBalancers(ApplicationId application, ZoneId zone); + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 9a489f8379d..8ca6373166b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -55,6 +55,7 @@ import com.yahoo.vespa.hosted.controller.concurrent.Once; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -115,9 +116,9 @@ public class ApplicationController { private final AccessControl accessControl; private final ConfigServer configServer; private final RoutingGenerator routingGenerator; + private final RoutingPolicies routingPolicies; private final Clock clock; private final BooleanFlag redirectLegacyDnsFlag; - private final DeploymentTrigger deploymentTrigger; ApplicationController(Controller controller, CuratorDb curator, @@ -130,6 +131,7 @@ public class ApplicationController { this.accessControl = accessControl; this.configServer = configServer; this.routingGenerator = routingGenerator; + this.routingPolicies = new RoutingPolicies(controller); this.clock = clock; this.redirectLegacyDnsFlag = Flags.REDIRECT_LEGACY_DNS_NAMES.bindTo(controller.flagSource()); @@ -327,6 +329,7 @@ public class ApplicationController { cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header app.rotation().map(RotationId::asString).ifPresent(cnames::add); + // Update application with information from application package if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally() @@ -422,6 +425,11 @@ public class ApplicationController { ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentId, deployOptions, cnames, rotationNames, applicationPackage.zippedContent()); + + // Refresh routing policies on successful deployment. At this point we can safely assume that the config server + // has allocated load balancers for the deployment. + routingPolicies.refresh(application, zone); + return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } @@ -643,9 +651,10 @@ public class ApplicationController { private LockedApplication deactivate(LockedApplication application, ZoneId zone) { try { configServer.deactivate(new DeploymentId(application.get().id(), zone)); - } - catch (NotFoundException ignored) { + } catch (NotFoundException ignored) { // ok; already gone + } finally { + routingPolicies.refresh(application.get().id(), zone); } return application.withoutDeploymentIn(zone); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index aec4c7a915c..3f5f8273922 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -53,7 +53,6 @@ public class ControllerMaintenance extends AbstractComponent { private final JobRunner jobRunner; private final ContactInformationMaintainer contactInformationMaintainer; private final CostReportMaintainer costReportMaintainer; - private final RoutingPolicyMaintainer routingPolicyMaintainer; private final ResourceMeterMaintainer resourceMeterMaintainer; private final NameServiceDispatcher nameServiceDispatcher; @@ -86,7 +85,6 @@ public class ControllerMaintenance extends AbstractComponent { osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); - routingPolicyMaintainer = new RoutingPolicyMaintainer(controller, Duration.ofMinutes(5), jobControl, curator); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); } @@ -116,7 +114,6 @@ public class ControllerMaintenance extends AbstractComponent { jobRunner.deconstruct(); contactInformationMaintainer.deconstruct(); costReportMaintainer.deconstruct(); - routingPolicyMaintainer.deconstruct(); resourceMeterMaintainer.deconstruct(); nameServiceDispatcher.deconstruct(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java new file mode 100644 index 00000000000..c9b46ff9dcf --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java @@ -0,0 +1,187 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.RoutingId; +import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicies { + + private final Controller controller; + private final CuratorDb db; + + public RoutingPolicies(Controller controller) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.db = controller.curator(); + try (Lock lock = db.lockRoutingPolicies()) { // Update serialized format + for (var policy : db.readRoutingPolicies().entrySet()) { + db.writeRoutingPolicies(policy.getKey(), policy.getValue()); + } + } + } + + + /** + * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if + * load balancers for given application have changed. + */ + public void refresh(ApplicationId application, ZoneId zone) { + var lbs = new LoadBalancers(application, zone, controller.applications().configServer() + .getLoadBalancers(application, zone)); + removeObsoleteEndpointsFromDns(lbs); + storePoliciesOf(lbs); + removeObsoletePolicies(lbs); + registerEndpointsInDns(lbs); + } + + /** Create global endpoints for given route, if any */ + private void registerEndpointsInDns(LoadBalancers loadBalancers) { + try (Lock lock = db.lockRoutingPolicies()) { + Map> routingTable = routingTableFrom(db.readRoutingPolicies(loadBalancers.application)); + + // Create DNS record for each routing ID + for (Map.Entry> routeEntry : routingTable.entrySet()) { + Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().rotation(), + controller.system()); + Set targets = routeEntry.getValue() + .stream() + .filter(policy -> policy.dnsZone().isPresent()) + .map(policy -> new AliasTarget(policy.canonicalName(), + policy.dnsZone().get(), + policy.zone())) + .collect(Collectors.toSet()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + } + } + } + + /** Store routing policies for given route */ + private void storePoliciesOf(LoadBalancers loadBalancers) { + try (Lock lock = db.lockRoutingPolicies()) { + Set policies = new LinkedHashSet<>(db.readRoutingPolicies(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer); + if (!policies.add(policy)) { + policies.remove(policy); + policies.add(policy); + } + } + db.writeRoutingPolicies(loadBalancers.application, policies); + } + } + + /** Create a policy for given load balancer and register a CNAME for it */ + private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { + RoutingPolicy routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, + loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancer.rotations()); + RecordName name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName()); + RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); + controller.nameServiceForwarder().createCname(name, data, Priority.normal); + return routingPolicy; + } + + /** Remove obsolete policies for given route and their CNAME records */ + private void removeObsoletePolicies(LoadBalancers loadBalancers) { + try (Lock lock = db.lockRoutingPolicies()) { + var allPolicies = new LinkedHashSet<>(db.readRoutingPolicies(loadBalancers.application)); + var removalCandidates = new HashSet<>(allPolicies); + var activeLoadBalancers = loadBalancers.list.stream() + .map(LoadBalancer::hostname) + .collect(Collectors.toSet()); + // Remove active load balancers and irrelevant zones from candidates + removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || + !policy.zone().equals(loadBalancers.zone)); + for (var policy : removalCandidates) { + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + allPolicies.remove(policy); + } + db.writeRoutingPolicies(loadBalancers.application, allPolicies); + } + } + + /** Remove unreferenced global endpoints for given route from DNS */ + private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers) { + try (Lock lock = db.lockRoutingPolicies()) { + var zonePolicies = db.readRoutingPolicies(loadBalancers.application).stream() + .filter(policy -> policy.zone().equals(loadBalancers.zone)) + .collect(Collectors.toUnmodifiableSet()); + var removalCandidates = routingTableFrom(zonePolicies).keySet(); + var activeRoutingIds = routingIdsFrom(loadBalancers.list); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + } + } + } + + /** Compute routing IDs from given load balancers */ + private static Set routingIdsFrom(List loadBalancers) { + Set routingIds = new LinkedHashSet<>(); + for (var loadBalancer : loadBalancers) { + for (var rotation : loadBalancer.rotations()) { + routingIds.add(new RoutingId(loadBalancer.application(), rotation)); + } + } + return Collections.unmodifiableSet(routingIds); + } + + /** Compute a routing table from given policies */ + private static Map> routingTableFrom(Set routingPolicies) { + var routingTable = new LinkedHashMap>(); + for (var policy : routingPolicies) { + for (var rotation : policy.rotations()) { + var id = new RoutingId(policy.owner(), rotation); + routingTable.putIfAbsent(id, new ArrayList<>()); + routingTable.get(id).add(policy); + } + } + return routingTable; + } + + /** Load balancers for a particular deployment */ + private static class LoadBalancers { + + private final ApplicationId application; + private final ZoneId zone; + private final List list; + + private LoadBalancers(ApplicationId application, ZoneId zone, List list) { + this.application = application; + this.zone = zone; + this.list = list; + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java deleted file mode 100644 index 0ddc24147ee..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java +++ /dev/null @@ -1,224 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.RotationName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.log.LogLevel; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.RoutingId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Maintains routing policies and their DNS records for all exclusive load balancers in this system. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicyMaintainer extends Maintainer { - - private static final Logger log = Logger.getLogger(RoutingPolicyMaintainer.class.getName()); - - private final NameServiceForwarder nameServiceForwarder; - private final CuratorDb db; - - public RoutingPolicyMaintainer(Controller controller, - Duration interval, - JobControl jobControl, - CuratorDb db) { - super(controller, interval, jobControl); - this.nameServiceForwarder = controller.nameServiceForwarder(); - this.db = db; - // Update serialized format - try (Lock lock = db.lockRoutingPolicies()) { - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - @Override - protected void maintain() { - Map> loadBalancers = findLoadBalancers(); - removeObsoleteEndpointsFromDns(loadBalancers); - storePolicies(loadBalancers); - removeObsoletePolicies(loadBalancers); - registerEndpointsInDns(); - } - - /** Find all exclusive load balancers in this system, grouped by deployment */ - private Map> findLoadBalancers() { - Map> result = new LinkedHashMap<>(); - for (ZoneId zone : controller().zoneRegistry().zones().controllerUpgraded().ids()) { - List loadBalancers = controller().applications().configServer().getLoadBalancers(zone); - for (LoadBalancer loadBalancer : loadBalancers) { - DeploymentId deployment = new DeploymentId(loadBalancer.application(), zone); - result.compute(deployment, (k, existing) -> { - if (existing == null) { - existing = new ArrayList<>(); - } - existing.add(loadBalancer); - return existing; - }); - } - } - return Collections.unmodifiableMap(result); - } - - /** Create global endpoints for all current routing policies */ - private void registerEndpointsInDns() { - try (Lock lock = db.lockRoutingPolicies()) { - Map> routingTable = routingTableFrom(db.readRoutingPolicies()); - - // Create DNS record for each routing ID - for (Map.Entry> route : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(route.getKey().application(), route.getKey().rotation(), - controller().system()); - Set targets = route.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - try { - nameServiceForwarder.createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to create or update DNS record for global rotation " + - endpoint.dnsName() + ". Retrying in " + maintenanceInterval(), e); - } - } - } - } - - /** Store routing policies for all load balancers */ - private void storePolicies(Map> loadBalancers) { - for (Map.Entry> entry : loadBalancers.entrySet()) { - ApplicationId application = entry.getKey().applicationId(); - ZoneId zone = entry.getKey().zoneId(); - try (Lock lock = db.lockRoutingPolicies()) { - Set policies = new LinkedHashSet<>(db.readRoutingPolicies(application)); - for (LoadBalancer loadBalancer : entry.getValue()) { - try { - RoutingPolicy policy = storePolicy(application, zone, loadBalancer); - if (!policies.add(policy)) { - policies.remove(policy); - policies.add(policy); - } - } catch (Exception e) { - log.log(LogLevel.WARNING, "Failed to create or update DNS record for load balancer " + - loadBalancer.hostname() + ". Retrying in " + maintenanceInterval(), - e); - } - } - db.writeRoutingPolicies(application, policies); - } - } - } - - /** Store policy for given load balancer and request a CNAME for it */ - private RoutingPolicy storePolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) { - RoutingPolicy routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, - loadBalancer.hostname(), loadBalancer.dnsZone(), - loadBalancer.rotations()); - RecordName name = RecordName.from(routingPolicy.endpointIn(controller().system()).dnsName()); - RecordData data = RecordData.fqdn(loadBalancer.hostname().value()); - nameServiceForwarder.createCname(name, data, Priority.normal); - return routingPolicy; - } - - /** Remove obsolete policies and their CNAME records */ - private void removeObsoletePolicies(Map> loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - var allPolicies = new HashMap<>(db.readRoutingPolicies()); - var removalCandidates = allPolicies.values().stream() - .flatMap(Collection::stream) - .collect(Collectors.toSet()); - var activeLoadBalancers = loadBalancers.values().stream() - .flatMap(Collection::stream) - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Keep active load balancers by removing them from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName())); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller().system()).dnsName(); - nameServiceForwarder.removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - // Remove stale policy from curator - var updatedPolicies = new LinkedHashSet<>(allPolicies.getOrDefault(policy.owner(), Set.of())); - updatedPolicies.remove(policy); - allPolicies.put(policy.owner(), updatedPolicies); - db.writeRoutingPolicies(policy.owner(), updatedPolicies); - } - } - } - - /** Remove DNS for global endpoints not referenced by given load balancers */ - private void removeObsoleteEndpointsFromDns(Map> loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - Set removalCandidates = routingTableFrom(db.readRoutingPolicies()).keySet(); - Set activeRoutingIds = routingIdsFrom(loadBalancers); - removalCandidates.removeAll(activeRoutingIds); - for (RoutingId id : removalCandidates) { - Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller().system()); - nameServiceForwarder.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } - } - } - - /** Compute routing IDs from given load balancers */ - private static Set routingIdsFrom(Map> loadBalancers) { - Set routingIds = new LinkedHashSet<>(); - for (List values : loadBalancers.values()) { - for (LoadBalancer loadBalancer : values) { - for (RotationName rotation : loadBalancer.rotations()) { - routingIds.add(new RoutingId(loadBalancer.application(), rotation)); - } - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Compute a routing table from given policies */ - private static Map> routingTableFrom(Map> routingPolicies) { - var flattenedPolicies = routingPolicies.values().stream().flatMap(Collection::stream).collect(Collectors.toSet()); - var routingTable = new LinkedHashMap>(); - for (var policy : flattenedPolicies) { - for (var rotation : policy.rotations()) { - var id = new RoutingId(policy.owner(), rotation); - routingTable.compute(id, (k, policies) -> { - if (policies == null) { - policies = new ArrayList<>(); - } - policies.add(policy); - return policies; - }); - } - } - return routingTable; - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 59d3f6dbb84..7dcb4aba138 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -294,7 +294,7 @@ public class ControllerTest { "app1--tenant1.global.vespa.yahooapis.com"), tester.configServer().rotationCnames().get(new DeploymentId(application.id(), deployment.zone()))); } - tester.updateDns(); + tester.flushDnsRequests(); assertEquals(3, tester.controllerTester().nameService().records().size()); Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 29bec7246ee..83b95ccc8b0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -5,6 +5,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import java.io.ByteArrayOutputStream; @@ -62,6 +63,10 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder region(RegionName regionName) { + return region(regionName.value()); + } + public ApplicationPackageBuilder region(String regionName) { environmentBody.append(" "); environmentBody.append(regionName); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index b6c7e369f07..a093aac430b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -140,8 +140,8 @@ public class DeploymentTester { readyJobTrigger().maintain(); } - /** Dispatch all pending name services requests */ - public void updateDns() { + /** Flush all pending name services requests */ + public void flushDnsRequests() { nameServiceDispatcher.run(); assertTrue("All name service requests dispatched", controller().curator().readNameServiceQueue().requests().isEmpty()); @@ -225,7 +225,7 @@ public class DeploymentTester { assertFalse(applications().require(application.id()).change().hasTargets()); } if (updateDnsAutomatically) { - updateDns(); + flushDnsRequests(); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index e201258c701..913c9a800c1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -204,14 +204,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return loadBalancers.getOrDefault(zone, Collections.emptyList()); } + @Override + public List getLoadBalancers(ApplicationId application, ZoneId zone) { + return getLoadBalancers(zone).stream() + .filter(lb -> lb.application().equals(application)) + .collect(Collectors.toUnmodifiableList()); + } + public void addLoadBalancers(ZoneId zone, List loadBalancers) { - this.loadBalancers.compute(zone, (k, existing) -> { - if (existing == null) { - existing = new ArrayList<>(); - } - existing.addAll(loadBalancers); - return existing; - }); + this.loadBalancers.putIfAbsent(zone, new ArrayList<>()); + this.loadBalancers.get(zone).addAll(loadBalancers); } public void removeLoadBalancers(ApplicationId application, ZoneId zone) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java index 13218cc2442..6a3eef5a142 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java @@ -104,7 +104,7 @@ public class DnsMaintainerTest { for (int i = 0; i < ControllerTester.availableRotations; i++) { maintainer.maintain(); } - tester.updateDns(); + tester.flushDnsRequests(); assertFalse("DNS record removed", findCname.apply("app1--tenant1.global.vespa.yahooapis.com").isPresent()); assertFalse("DNS record removed", findCname.apply("app1--tenant1.global.vespa.oath.cloud").isPresent()); assertFalse("DNS record removed", findCname.apply("app1.tenant1.global.vespa.yahooapis.com").isPresent()); @@ -124,7 +124,7 @@ public class DnsMaintainerTest { // One record is removed per run for (int i = 1; i <= staleTotal*2; i++) { maintainer.run(); - tester.updateDns(); + tester.flushDnsRequests(); assertEquals(Math.max(staleTotal - i, 0), records().size()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index b18c39f4042..58f35c0ac05 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -255,7 +255,7 @@ public class MetricsReporterTest { reporter.maintain(); assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); - tester.updateDns(); + tester.flushDnsRequests(); reporter.maintain(); assertEquals("Queue consumed", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java new file mode 100644 index 00000000000..1cfb18da851 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java @@ -0,0 +1,237 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RotationName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author mortent + * @author mpolden + */ +public class RoutingPoliciesTest { + + private final DeploymentTester tester = new DeploymentTester(); + + private final Application app1 = tester.createApplication("app1", "tenant1", 1, 1L); + private final Application app2 = tester.createApplication("app2", "tenant1", 1, 1L); + + private final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); + private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); + private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); + + private final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .build(); + + @Test + public void maintains_global_routing_policies() { + int buildNumber = 42; + int clustersPerZone = 2; + // Cluster 0 is member of 2 global rotations + Map> rotations = Map.of(0, Set.of(RotationName.from("r0"), RotationName.from("r1"))); + provisionLoadBalancers(clustersPerZone, rotations, app1.id(), zone1, zone2); + + // Creates alias records for cluster0 + tester.deployCompletely(app1, applicationPackage, ++buildNumber); + Supplier> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app1.tenant1.global.vespa.oath.cloud")); + Supplier> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r1.app1.tenant1.global.vespa.oath.cloud")); + assertEquals(2, records1.get().size()); + assertEquals(records1.get().size(), records2.get().size()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records1.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records2.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records2.get().get(1).data().asString()); + assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() + .rotationEndpointsIn(SystemName.main).asList().size()); + + // Applications gains a new deployment + ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region(zone1.region()) + .region(zone2.region()) + .region(zone3.region()) + .build(); + int numberOfDeployments = 3; + provisionLoadBalancers(clustersPerZone, rotations, app1.id(), zone3); + tester.deployCompletely(app1, updatedApplicationPackage, ++buildNumber); + + // Cluster in new deployment is added to the rotation + assertEquals(numberOfDeployments, records1.get().size()); + assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records1.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-east-3/dns-zone-1/prod.us-east-3", records1.get().get(1).data().asString()); + assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records1.get().get(2).data().asString()); + + // Another application is deployed + Supplier> records3 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app2.tenant1.global.vespa.oath.cloud")); + provisionLoadBalancers(1, Map.of(0, Set.of(RotationName.from("r0"))), app2.id(), zone1, zone2); + tester.deployCompletely(app2, applicationPackage); + assertEquals(2, records3.get().size()); + assertEquals("lb-0--tenant1:app2:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records3.get().get(0).data().asString()); + assertEquals("lb-0--tenant1:app2:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records3.get().get(1).data().asString()); + + // All rotations for app1 are removed + provisionLoadBalancers(clustersPerZone, Map.of(), app1.id(), zone1, zone2, zone3); + tester.deployCompletely(app1, updatedApplicationPackage, ++buildNumber); + assertEquals(List.of(), records1.get()); + Set policies = tester.controller().curator().readRoutingPolicies(app1.id()); + assertEquals(clustersPerZone * numberOfDeployments, policies.size()); + assertTrue("Rotation membership is removed from all policies", + policies.stream().allMatch(policy -> policy.rotations().isEmpty())); + assertEquals("Rotations for " + app2 + " are not removed", 2, records3.get().size()); + } + + @Test + public void maintains_routing_policies_per_zone() { + // Deploy application + int clustersPerZone = 2; + int buildNumber = 42; + provisionLoadBalancers(clustersPerZone, app1.id(), zone1, zone2); + tester.deployCompletely(app1, applicationPackage, ++buildNumber); + + // Deployment creates records and policies for all clusters in all zones + Set expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, recordNames()); + assertEquals(4, policies(app1).size()); + + // Next deploy does nothing + tester.deployCompletely(app1, applicationPackage, ++buildNumber); + assertEquals(expectedRecords, recordNames()); + assertEquals(4, policies(app1).size()); + + // Add 1 cluster in each zone and deploy + provisionLoadBalancers(clustersPerZone + 1, app1.id(), zone1, zone2); + tester.deployCompletely(app1, applicationPackage, ++buildNumber); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, recordNames()); + assertEquals(6, policies(app1).size()); + + // Deploy another application + provisionLoadBalancers(clustersPerZone, app2.id(), zone1, zone2); + tester.deployCompletely(app2, applicationPackage, ++buildNumber); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, recordNames()); + assertEquals(4, policies(app2).size()); + + // Deploy removes cluster from app1 + provisionLoadBalancers(clustersPerZone, app1.id(), zone1, zone2); + tester.deployCompletely(app1, applicationPackage, ++buildNumber); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, recordNames()); + + // Remove app2 completely + tester.controller().applications().require(app2.id()).deployments().keySet() + .forEach(zone -> { + tester.configServer().removeLoadBalancers(app2.id(), zone); + tester.controller().applications().deactivate(app2.id(), zone); + }); + tester.flushDnsRequests(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, recordNames()); + assertTrue("Removes stale routing policies " + app2, tester.controller().applications().routingPolicies(app2.id()).isEmpty()); + assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies(app1.id()).size()); + } + + private Set policies(Application application) { + return tester.controller().curator().readRoutingPolicies(application.id()); + } + + private Set recordNames() { + return tester.controllerTester().nameService().records().stream() + .map(Record::name) + .map(RecordName::asString) + .collect(Collectors.toSet()); + } + + private void provisionLoadBalancers(int clustersPerZone, Map> clusterRotations, ApplicationId application, ZoneId... zones) { + for (ZoneId zone : zones) { + tester.configServer().removeLoadBalancers(application, zone); + tester.configServer().addLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone, clusterRotations)); + } + } + + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + provisionLoadBalancers(clustersPerZone, Map.of(), application, zones); + } + + private static List createLoadBalancers(ZoneId zone, ApplicationId application, int count, + Map> clusterRotations) { + List loadBalancers = new ArrayList<>(); + for (int i = 0; i < count; i++) { + Set rotations = clusterRotations.getOrDefault(i, Collections.emptySet()); + loadBalancers.add( + new LoadBalancer("LB-" + i + "-Z-" + zone.value(), + application, + ClusterSpec.Id.from("c" + i), + HostName.from("lb-" + i + "--" + application.serializedForm() + + "--" + zone.value()), + Optional.of("dns-zone-1"), + rotations)); + } + return loadBalancers; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java deleted file mode 100644 index 14d5dc4e7c3..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java +++ /dev/null @@ -1,250 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RotationName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.BuildJob; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Test; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author mortent - * @author mpolden - */ -public class RoutingPolicyMaintainerTest { - - private final DeploymentTester tester = new DeploymentTester(); - private final Application app1 = tester.createApplication("app1", "tenant1", 1, 1L); - private final Application app2 = tester.createApplication("app2", "tenant1", 1, 1L); - - private final RoutingPolicyMaintainer maintainer = new RoutingPolicyMaintainer(tester.controller(), Duration.ofHours(12), - new JobControl(new MockCuratorDb()), - tester.controllerTester().curator()); - private final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("us-west-1") - .region("us-central-1") - .build(); - - @Test - public void maintains_global_routing_policies() { - int clustersPerZone = 2; - tester.deployCompletely(app1, applicationPackage); - // Cluster is member of 2 global rotations - Map> rotations = Map.of(0, Set.of(RotationName.from("r0"), RotationName.from("r1"))); - provisionLoadBalancers(app1, clustersPerZone, rotations); - - // Creates alias records for cluster0 - maintain(); - Supplier> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app1.tenant1.global.vespa.oath.cloud")); - Supplier> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r1.app1.tenant1.global.vespa.oath.cloud")); - assertEquals(2, records1.get().size()); - assertEquals(records1.get().size(), records2.get().size()); - assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records1.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records1.get().get(1).data().asString()); - assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records2.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records2.get().get(1).data().asString()); - assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() - .rotationEndpointsIn(SystemName.main).asList().size()); - - // Applications gains a new deployment - ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .region("us-west-1") - .region("us-central-1") - .region("us-east-3") - .build(); - int numberOfDeployments = 3; - tester.deployCompletely(app1, updatedApplicationPackage, BuildJob.defaultBuildNumber + 1); - - // Cluster in new deployment is added to the rotation - provisionLoadBalancers(app1, 2, rotations); - maintain(); - assertEquals(numberOfDeployments, records1.get().size()); - assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records1.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app1:default--prod.us-east-3/dns-zone-1/prod.us-east-3", records1.get().get(1).data().asString()); - assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records1.get().get(2).data().asString()); - - // Another application is deployed - Supplier> records3 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0.app2.tenant1.global.vespa.oath.cloud")); - tester.deployCompletely(app2, applicationPackage); - provisionLoadBalancers(app2, 1, Map.of(0, Set.of(RotationName.from("r0")))); - maintain(); - assertEquals(2, records3.get().size()); - assertEquals("lb-0--tenant1:app2:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records3.get().get(0).data().asString()); - assertEquals("lb-0--tenant1:app2:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records3.get().get(1).data().asString()); - - // All rotations for app1 are removed - provisionLoadBalancers(app1, clustersPerZone, Collections.emptyMap()); - maintain(); - assertEquals(List.of(), records1.get()); - Set policies = tester.controller().curator().readRoutingPolicies(app1.id()); - assertEquals(clustersPerZone * numberOfDeployments, policies.size()); - assertTrue("Rotation membership is removed from all policies", - policies.stream().allMatch(policy -> policy.rotations().isEmpty())); - assertEquals("Rotations for " + app2 + " are not removed", 2, records3.get().size()); - } - - @Test - public void maintains_routing_policies_per_zone() { - // Deploy application - int clustersPerZone = 2; - tester.deployCompletely(app1, applicationPackage); - provisionLoadBalancers(app1, clustersPerZone); - - // Creates records and policies for all clusters in all zones - maintain(); - Set expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, recordNames()); - assertEquals(4, policies(app1).size()); - - // Next run does nothing - maintain(); - assertEquals(expectedRecords, recordNames()); - assertEquals(4, policies(app1).size()); - - // Add 1 cluster in each zone - provisionLoadBalancers(app1, clustersPerZone + 1); - maintain(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, recordNames()); - assertEquals(6, policies(app1).size()); - - // Add another application - tester.deployCompletely(app2, applicationPackage); - provisionLoadBalancers(app2, clustersPerZone); - maintain(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, recordNames()); - assertEquals(4, policies(app2).size()); - - // Remove cluster from app1 - provisionLoadBalancers(app1, clustersPerZone); - maintain(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", - "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, recordNames()); - - // Remove app2 completely - tester.controller().applications().require(app2.id()).deployments().keySet() - .forEach(zone -> { - tester.controller().applications().deactivate(app2.id(), zone); - tester.configServer().removeLoadBalancers(app2.id(), zone); - }); - maintain(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" - ); - assertEquals(expectedRecords, recordNames()); - assertTrue("Removes stale routing policies " + app2, tester.controller().applications().routingPolicies(app2.id()).isEmpty()); - assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies(app1.id()).size()); - } - - private void maintain() { - maintainer.run(); - tester.updateDns(); - } - - private Set policies(Application application) { - return tester.controller().curator().readRoutingPolicies(application.id()); - } - - private Set recordNames() { - return tester.controllerTester().nameService().records().stream() - .map(Record::name) - .map(RecordName::asString) - .collect(Collectors.toSet()); - } - - private void provisionLoadBalancers(Application application, int clustersPerZone, Map> clusterRotations) { - tester.controller().applications().require(application.id()) - .deployments().keySet() - .forEach(zone -> tester.configServer().removeLoadBalancers(application.id(), zone)); - tester.controller().applications().require(application.id()) - .deployments().keySet() - .forEach(zone -> tester.configServer() - .addLoadBalancers(zone, createLoadBalancers(zone, application.id(), clustersPerZone, clusterRotations))); - } - - private void provisionLoadBalancers(Application application, int clustersPerZone) { - provisionLoadBalancers(application, clustersPerZone, Collections.emptyMap()); - } - - private static List createLoadBalancers(ZoneId zone, ApplicationId application, int count, - Map> clusterRotations) { - List loadBalancers = new ArrayList<>(); - for (int i = 0; i < count; i++) { - Set rotations = clusterRotations.getOrDefault(i, Collections.emptySet()); - loadBalancers.add( - new LoadBalancer("LB-" + i + "-Z-" + zone.value(), - application, - ClusterSpec.Id.from("c" + i), - HostName.from("lb-" + i + "--" + application.serializedForm() + - "--" + zone.value()), - Optional.of("dns-zone-1"), - rotations)); - } - return loadBalancers; - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 6b20adf835e..6218d9d04f0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -51,9 +51,6 @@ { "name": "ResourceMeterMaintainer" }, - { - "name": "RoutingPolicyMaintainer" - }, { "name": "SystemUpgrader" }, -- cgit v1.2.3 From e2ddffe8251f732d9f2b787d2e91d94b8ca547fa Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 23 May 2019 13:03:25 +0200 Subject: Listen to /api/zone/v1 with the zone api handler --- .../hosted/controller/api/role/PathGroup.java | 3 ++- .../controller/restapi/zone/v1/ZoneApiHandler.java | 4 +++- .../restapi/ControllerContainerCloudTest.java | 7 ++++++ .../restapi/ControllerContainerTest.java | 8 +++---- .../controller/restapi/zone/v1/ZoneApiTest.java | 28 +++++++++++++--------- 5 files changed, 33 insertions(+), 17 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index d0f9b769494..ec44b5605b6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -134,7 +134,8 @@ enum PathGroup { "/statuspage/v1/{*}"), /** Paths providing public information. */ - publicInfo("/badge/v1/{*}", + publicInfo(Optional.of("/api"), + "/badge/v1/{*}", "/zone/v1/{*}"); final List pathSpecs; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index fcf01f461a1..b115e659c28 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -28,6 +28,8 @@ import java.util.stream.Collectors; @SuppressWarnings("unused") public class ZoneApiHandler extends LoggingRequestHandler { + private static final String OPTIONAL_PREFIX = "/api"; + private final ZoneRegistry zoneRegistry; public ZoneApiHandler(LoggingRequestHandler.Context parentCtx, ZoneRegistry zoneRegistry) { @@ -54,7 +56,7 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse get(HttpRequest request) { - Path path = new Path(request.getUri()); + Path path = new Path(request.getUri(), OPTIONAL_PREFIX); if (path.matches("/zone/v1")) { return root(request); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java index 4f068451d24..102326d0a5e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerCloudTest.java @@ -34,6 +34,13 @@ public class ControllerContainerCloudTest extends ControllerContainerTest { " http://*/api/application/v4/*\n" + " \n" + + " \n" + + " http://*/zone/v1\n" + + " http://*/zone/v1/*\n" + + " http://*/api/zone/v1\n" + + " http://*/api/zone/v1/*\n" + + " \n" + + " \n" + " http://*/user/v1/*\n" + " http://*/api/user/v1/*\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 8e12268c197..e974f55c9f6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -106,10 +106,6 @@ public class ControllerContainerTest { " \n" + " http://*/cost/v1/*\n" + " \n" + - " \n" + - " http://*/zone/v1\n" + - " http://*/zone/v1/*\n" + - " \n" + " \n" + " http://*/zone/v2\n" + " http://*/zone/v2/*\n" + @@ -132,6 +128,10 @@ public class ControllerContainerTest { " \n" + " http://*/athenz/v1/*\n" + " \n" + + " \n" + + " http://*/zone/v1\n" + + " http://*/zone/v1/*\n" + + " \n" + " \n" + " \n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index 9c853f211ed..ca4061f8376 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -4,27 +4,29 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; +import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; import org.junit.Test; import java.io.File; import java.util.List; +import java.util.Set; /** * @author mpolden */ -public class ZoneApiTest extends ControllerContainerTest { +public class ZoneApiTest extends ControllerContainerCloudTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/"; - private static final List zones = List.of( - ZoneId.from(Environment.prod, RegionName.from("us-north-1")), - ZoneId.from(Environment.dev, RegionName.from("us-north-2")), - ZoneId.from(Environment.test, RegionName.from("us-north-3")), - ZoneId.from(Environment.staging, RegionName.from("us-north-4")) - ); + private static final List zones = List.of(ZoneId.from(Environment.prod, RegionName.from("us-north-1")), + ZoneId.from(Environment.dev, RegionName.from("us-north-2")), + ZoneId.from(Environment.test, RegionName.from("us-north-3")), + ZoneId.from(Environment.staging, RegionName.from("us-north-4"))); + private static final Set everyone = Set.of(Role.everyone()); private ContainerControllerTester tester; @@ -40,22 +42,26 @@ public class ZoneApiTest extends ControllerContainerTest { @Test public void test_requests() { // GET /zone/v1 - tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v1"), + tester.containerTester().assertResponse(request("/zone/v1") + .roles(everyone), new File("root.json")); // GET /zone/v1/environment/prod - tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v1/environment/prod"), + tester.containerTester().assertResponse(request("/zone/v1/environment/prod") + .roles(everyone), new File("prod.json")); // GET /zone/v1/environment/dev/default - tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v1/environment/dev/default"), + tester.containerTester().assertResponse(request("/api/zone/v1/environment/dev/default") + .roles(everyone), new File("default-for-region.json")); } @Test public void test_invalid_requests() { // GET /zone/v1/environment/prod/default: No default region - tester.containerTester().assertResponse(authenticatedRequest("http://localhost:8080/zone/v1/environment/prod/default"), + tester.containerTester().assertResponse(request("/zone/v1/environment/prod/default") + .roles(everyone), new File("no-default-region.json"), 400); } -- cgit v1.2.3 From 5e6408d3775e2ea251d78f00abef4723b37f352a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 23 May 2019 14:32:19 +0200 Subject: Remove unused @Inject --- .../java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java | 1 - 1 file changed, 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java index fb1e0599a4c..f13c31de5d7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java @@ -34,7 +34,6 @@ public class JobRunner extends Maintainer { private final ExecutorService executors; private final StepRunner runner; - @Inject public JobRunner(Controller controller, Duration duration, JobControl jobControl) { this(controller, duration, jobControl, Executors.newFixedThreadPool(32), new InternalStepRunner(controller)); } -- cgit v1.2.3 From e633f02a605128582bc7719defe0cb7c8f931de1 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 23 May 2019 14:32:59 +0200 Subject: Avoid NPE trying to render something we have no data for --- .../restapi/application/JobControllerApiHandlerHelper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index d402f150725..5e854f4cfad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -94,9 +94,11 @@ class JobControllerApiHandlerHelper { Slime slime = new Slime(); Cursor responseObject = slime.setObject(); - Cursor lastVersionsObject = responseObject.setObject("lastVersions"); - lastPlatformToSlime(lastVersionsObject.setObject("platform"), controller, application, change, steps); - lastApplicationToSlime(lastVersionsObject.setObject("application"), application, change, steps, controller); + if (application.deploymentJobs().statusOf(component).flatMap(JobStatus::lastSuccess).isPresent()) { + Cursor lastVersionsObject = responseObject.setObject("lastVersions"); + lastPlatformToSlime(lastVersionsObject.setObject("platform"), controller, application, change, steps); + lastApplicationToSlime(lastVersionsObject.setObject("application"), application, change, steps, controller); + } if ( ! change.isEmpty()) { Cursor deployingObject = responseObject.setObject("deploying"); -- cgit v1.2.3 From 39c819c409776fe21c370a33773f57b034d90961 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 23 May 2019 14:34:46 +0200 Subject: Render info about last dev/perf deployment jobs in overview --- .../api/integration/deployment/JobType.java | 6 ++++++ .../application/JobControllerApiHandlerHelper.java | 11 ++++++++++ .../JobControllerApiHandlerHelperTest.java | 14 ++++++++++++ .../application/responses/dev-overview.json | 25 ++++++++++++++++++++++ .../restapi/application/responses/jobs.json | 4 +++- .../restapi/application/responses/overview.json | 3 ++- 6 files changed, 61 insertions(+), 2 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 7bbb7886306..83e70a3429f 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -7,8 +7,10 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.cd; @@ -108,6 +110,10 @@ public enum JobType { return zones.get(system); } + public static List allIn(SystemName system) { + return Stream.of(values()).filter(job -> job.zones.containsKey(system)).collect(Collectors.toUnmodifiableList()); + } + /** Returns whether this is a production job */ public boolean isProduction() { return environment() == Environment.prod; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index 5e854f4cfad..ef7510ad18c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -134,6 +134,17 @@ class JobControllerApiHandlerHelper { running, baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize()); }); + + Cursor devJobsObject = responseObject.setObject("devJobs"); + for (JobType type : JobType.allIn(controller.system())) + if ( type.environment() != null + && type.environment().isManuallyDeployed() + && application.deployments().containsKey(type.zone(controller.system()))) + controller.jobController().last(application.id(), type) + .ifPresent(last -> runToSlime(devJobsObject.setObject(type.jobName()), + last, + baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize())); + return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java index 8580677431c..616db640132 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java @@ -120,6 +120,20 @@ public class JobControllerApiHandlerHelperTest { assertResponse(JobControllerApiHandlerHelper.runResponse(tester.jobs().runs(appId, devAwsUsEast2a), URI.create("https://some.url:43/root")), "dev-aws-us-east-2a-runs.json"); } + @Test + public void testDevResponses() { + InternalDeploymentTester tester = new InternalDeploymentTester(); + tester.clock().setInstant(Instant.EPOCH); + + ZoneId zone = JobType.devUsEast1.zone(tester.tester().controller().system()); + tester.jobs().deploy(appId, JobType.devUsEast1, Optional.empty(), applicationPackage); + tester.configServer().convergeServices(appId, zone); + tester.setEndpoints(appId, zone); + tester.runner().run(); + + assertResponse(JobControllerApiHandlerHelper.jobTypeResponse(tester.tester().controller(), appId, URI.create("https://some.url:43/root")), "dev-overview.json"); + } + private void compare(HttpResponse response, String expected) throws JSONException, IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); response.render(baos); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json new file mode 100644 index 00000000000..c845d31a5fc --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -0,0 +1,25 @@ +{ + "deployments": [], + "jobs": {}, + "devJobs": { + "dev-us-east-1": { + "id": 1, + "status": "success", + "start": 0, + "end": 0, + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, + "log": "https://some.url:43/root/dev-us-east-1/run/1" + } + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index 72dd13474dc..a5e4dd23079 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -128,5 +128,7 @@ ], "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1" } - } + }, + "devJobs": {} } + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json index 84866650afc..a8ec9295868 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/overview.json @@ -739,5 +739,6 @@ ], "url": "https://some.url:43/root/production-us-east-3" } - } + }, + "devJobs": {} } -- cgit v1.2.3 From 34c4932d15693496ec53d6dd77dfbe2e989e3943 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 23 May 2019 14:35:03 +0200 Subject: Avoid uncontrollable multi-threaded execution in unit test --- .../hosted/controller/restapi/application/ApplicationApiTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 4ec075c3c16..a12a2b5e57a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -133,6 +133,7 @@ public class ApplicationApiTest extends ControllerContainerTest { @Test public void testApplicationApi() { tester.computeVersionStatus(); + tester.controller().jobController().setRunner(__ -> { }); // Avoid uncontrollable, multi-threaded job execution createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); // (Necessary but not provided in this API) @@ -658,11 +659,6 @@ public class ApplicationApiTest extends ControllerContainerTest { .screwdriverIdentity(SCREWDRIVER_ID), "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); - // DELETE the deployment to dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default", DELETE) - .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default"); - // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), -- cgit v1.2.3 From 0ed7a5bac946d0e969488c41b914f319e992bd33 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 24 May 2019 09:16:29 +0200 Subject: Move read methods to RoutingPolicies --- .../hosted/controller/ApplicationController.java | 13 ++------ .../controller/deployment/InternalStepRunner.java | 3 +- .../controller/deployment/JobController.java | 3 +- .../controller/maintenance/RoutingPolicies.java | 38 +++++++++++++++------- .../restapi/application/ApplicationApiHandler.java | 4 +-- .../maintenance/RoutingPoliciesTest.java | 6 ++-- 6 files changed, 35 insertions(+), 32 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 8ca6373166b..8231de59249 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -48,7 +48,6 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.JobStatus.JobRun; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.concurrent.Once; @@ -714,16 +713,8 @@ public class ApplicationController { return rotationRepository; } - /** Returns all known routing policies for given application */ - public Set routingPolicies(ApplicationId application) { - return curator.readRoutingPolicies(application); - } - - /** Returns all known routing policies for given deployment */ - public Set routingPolicies(DeploymentId deployment) { - return curator.readRoutingPolicies(deployment.applicationId()).stream() - .filter(policy -> policy.zone().equals(deployment.zoneId())) - .collect(Collectors.toUnmodifiableSet()); + public RoutingPolicies routingPolicies() { + return routingPolicies; } /** Sort given list of applications by application ID */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 83a347819f2..7a42456f2ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -38,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; @@ -625,7 +624,7 @@ public class InternalStepRunner implements StepRunner { for (ZoneId zone : zones) { controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) .filter(endpoints -> ! endpoints.isEmpty()) - .or(() -> Optional.of(controller.applications().routingPolicies(new DeploymentId(id, zone)).stream() + .or(() -> Optional.of(controller.applications().routingPolicies().get(id, zone).stream() .map(policy -> policy.endpointIn(controller.system()).url()) .collect(Collectors.toUnmodifiableList())) .filter(endpoints -> ! endpoints.isEmpty())) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 5241267460c..5bb9686ce0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -22,7 +22,6 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.persistence.BufferedLogStore; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -410,7 +409,7 @@ public class JobController { DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); return controller.applications().getDeploymentEndpoints(testerId) .flatMap(uris -> uris.stream().findAny()) - .or(() -> controller.applications().routingPolicies(testerId).stream() + .or(() -> controller.applications().routingPolicies().get(testerId).stream() .findAny() .map(policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java index c9b46ff9dcf..07cbb34d378 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java @@ -3,8 +3,8 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -41,13 +41,29 @@ public class RoutingPolicies { public RoutingPolicies(Controller controller) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.db = controller.curator(); - try (Lock lock = db.lockRoutingPolicies()) { // Update serialized format + try (var lock = db.lockRoutingPolicies()) { // Update serialized format for (var policy : db.readRoutingPolicies().entrySet()) { db.writeRoutingPolicies(policy.getKey(), policy.getValue()); } } } + /** Read all known routing policies for given application */ + public Set get(ApplicationId application) { + return db.readRoutingPolicies(application); + } + + /** Read all known routing policies for given deployment */ + public Set get(DeploymentId deployment) { + return get(deployment.applicationId(), deployment.zoneId()); + } + + /** Read all known routing policies for given deployment */ + public Set get(ApplicationId application, ZoneId zone) { + return db.readRoutingPolicies(application).stream() + .filter(policy -> policy.zone().equals(zone)) + .collect(Collectors.toUnmodifiableSet()); + } /** * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if @@ -64,8 +80,8 @@ public class RoutingPolicies { /** Create global endpoints for given route, if any */ private void registerEndpointsInDns(LoadBalancers loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - Map> routingTable = routingTableFrom(db.readRoutingPolicies(loadBalancers.application)); + try (var lock = db.lockRoutingPolicies()) { + Map> routingTable = routingTableFrom(get(loadBalancers.application)); // Create DNS record for each routing ID for (Map.Entry> routeEntry : routingTable.entrySet()) { @@ -85,8 +101,8 @@ public class RoutingPolicies { /** Store routing policies for given route */ private void storePoliciesOf(LoadBalancers loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - Set policies = new LinkedHashSet<>(db.readRoutingPolicies(loadBalancers.application)); + try (var lock = db.lockRoutingPolicies()) { + Set policies = new LinkedHashSet<>(get(loadBalancers.application)); for (LoadBalancer loadBalancer : loadBalancers.list) { RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer); if (!policies.add(policy)) { @@ -111,8 +127,8 @@ public class RoutingPolicies { /** Remove obsolete policies for given route and their CNAME records */ private void removeObsoletePolicies(LoadBalancers loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - var allPolicies = new LinkedHashSet<>(db.readRoutingPolicies(loadBalancers.application)); + try (var lock = db.lockRoutingPolicies()) { + var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); var removalCandidates = new HashSet<>(allPolicies); var activeLoadBalancers = loadBalancers.list.stream() .map(LoadBalancer::hostname) @@ -131,10 +147,8 @@ public class RoutingPolicies { /** Remove unreferenced global endpoints for given route from DNS */ private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers) { - try (Lock lock = db.lockRoutingPolicies()) { - var zonePolicies = db.readRoutingPolicies(loadBalancers.application).stream() - .filter(policy -> policy.zone().equals(loadBalancers.zone)) - .collect(Collectors.toUnmodifiableSet()); + try (var lock = db.lockRoutingPolicies()) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone); var removalCandidates = routingTableFrom(zonePolicies).keySet(); var activeRoutingIds = routingIdsFrom(loadBalancers.list); removalCandidates.removeAll(activeRoutingIds); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 872f4d5d1af..f5b6f284d6b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -499,7 +499,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { application.rotation().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations - Set routingPolicies = controller.applications().routingPolicies(application.id()); + Set routingPolicies = controller.applications().routingPolicies().get(application.id()); for (RoutingPolicy policy : routingPolicies) { policy.rotationEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) @@ -585,7 +585,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add endpoint(s) defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies(deploymentId)) { + for (var policy : controller.applications().routingPolicies().get(deploymentId)) { Cursor endpointObject = endpointArray.addObject(); Endpoint endpoint = policy.endpointIn(controller.system()); endpointObject.setString("cluster", policy.cluster().value()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java index 1cfb18da851..d94c40e6b99 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java @@ -68,7 +68,7 @@ public class RoutingPoliciesTest { assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records1.get().get(1).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", records2.get().get(0).data().asString()); assertEquals("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1", records2.get().get(1).data().asString()); - assertEquals(2, tester.controller().applications().routingPolicies(app1.id()).iterator().next() + assertEquals(2, tester.controller().applications().routingPolicies().get(app1.id()).iterator().next() .rotationEndpointsIn(SystemName.main).asList().size()); // Applications gains a new deployment @@ -191,8 +191,8 @@ public class RoutingPoliciesTest { "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, recordNames()); - assertTrue("Removes stale routing policies " + app2, tester.controller().applications().routingPolicies(app2.id()).isEmpty()); - assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies(app1.id()).size()); + assertTrue("Removes stale routing policies " + app2, tester.controller().applications().routingPolicies().get(app2.id()).isEmpty()); + assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies().get(app1.id()).size()); } private Set policies(Application application) { -- cgit v1.2.3 From c6fc221d7831b3751111842f73de655e52c161a8 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Sat, 25 May 2019 13:49:02 +0200 Subject: Handle tenant type CLOUD --- .../hosted/controller/api/application/v4/model/TenantType.java | 3 ++- .../controller/restapi/application/ApplicationApiHandler.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java index 9f89da717be..0f04a31a9ed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/TenantType.java @@ -6,5 +6,6 @@ package com.yahoo.vespa.hosted.controller.api.application.v4.model; */ public enum TenantType { USER, - ATHENS + ATHENS, + CLOUD } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index f5b6f284d6b..d231b034e8a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1144,7 +1144,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void toSlime(Cursor object, Tenant tenant, HttpRequest request) { object.setString("tenant", tenant.name().value()); - object.setString("type", tentantType(tenant)); + object.setString("type", tenantType(tenant)); switch (tenant.type()) { case athenz: AthenzTenant athenzTenant = (AthenzTenant) tenant; @@ -1181,7 +1181,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private void tenantInTenantsListToSlime(Tenant tenant, URI requestURI, Cursor object) { object.setString("tenant", tenant.name().value()); Cursor metaData = object.setObject("metaData"); - metaData.setString("type", tentantType(tenant)); + metaData.setString("type", tenantType(tenant)); switch (tenant.type()) { case athenz: AthenzTenant athenzTenant = (AthenzTenant) tenant; @@ -1389,7 +1389,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return ImmutableSet.of("all", "true", "deployment").contains(request.getProperty("recursive")); } - private static String tentantType(Tenant tenant) { + private static String tenantType(Tenant tenant) { switch (tenant.type()) { case user: return "USER"; case athenz: return "ATHENS"; -- cgit v1.2.3 From b4c7bc329f5400e241daf625f9926dd4c89705fb Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Mon, 27 May 2019 16:00:58 +0200 Subject: Make rotations be List<> instead of Optional<> This allows us to have multiple rotations assigned to service, which in the future allows multiple rotations per application. --- .../yahoo/vespa/hosted/controller/Application.java | 21 ++++++---- .../hosted/controller/ApplicationController.java | 2 +- .../vespa/hosted/controller/LockedApplication.java | 49 +++++++++++----------- .../persistence/ApplicationSerializer.java | 30 ++++++++++--- .../restapi/application/ApplicationApiHandler.java | 7 ++-- .../controller/rotation/RotationRepository.java | 10 ++--- .../vespa/hosted/controller/ControllerTest.java | 2 +- .../persistence/ApplicationSerializerTest.java | 4 +- .../rotation/RotationRepositoryTest.java | 12 +++--- 9 files changed, 81 insertions(+), 56 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 84e15deea4c..1bae9e0ccd4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -18,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -56,7 +57,7 @@ public class Application { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional pemDeployKey; - private final Optional rotation; + private final List rotations; private final Map rotationStatus; /** Creates an empty application */ @@ -65,7 +66,7 @@ public class Application { new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), - Optional.empty(), Optional.empty(), Collections.emptyMap()); + Optional.empty(), Collections.emptyList(), Collections.emptyMap()); } /** Used from persistence layer: Do not use */ @@ -73,18 +74,18 @@ public class Application { List deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - Optional rotation, Map rotationStatus) { + List rotations, Map rotationStatus) { this(id, createdAt, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())), deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } Application(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - Optional rotation, Map rotationStatus) { + List rotations, Map rotationStatus) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -98,7 +99,7 @@ public class Application { this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.pemDeployKey = pemDeployKey; - this.rotation = Objects.requireNonNull(rotation, "rotation cannot be null"); + this.rotations = Objects.requireNonNull(rotations, "rotations cannot be null"); this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); } @@ -195,13 +196,15 @@ public class Application { } /** Returns the global rotation id of this, if present */ - public Optional rotation() { - return rotation; + public List rotations() { + return Collections.unmodifiableList(rotations); } /** Returns the default global endpoints for this in given system */ public EndpointList endpointsIn(SystemName system) { - if (rotation.isEmpty()) return EndpointList.EMPTY; + // TODO: Do we need to change something here? .defaultGlobalId seems like it is + // TODO: making some assumptions on naming. + if (rotations.isEmpty()) return EndpointList.EMPTY; return EndpointList.defaultGlobal(id, system); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 8ca6373166b..5c80e12dab4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -328,7 +328,7 @@ public class ApplicationController { // Include global DNS names cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header - app.rotation().map(RotationId::asString).ifPresent(cnames::add); + app.rotations().stream().map(RotationId::asString).forEach(cnames::add); // Update application with information from application package if ( ! preferOldestVersion diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 943ede5197b..e8d69f8a577 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -55,7 +56,7 @@ public class LockedApplication { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional pemDeployKey; - private final Optional rotation; + private final List rotations; private final Map rotationStatus; /** @@ -70,7 +71,7 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), - application.pemDeployKey(), application.rotation(), application.rotationStatus()); + application.pemDeployKey(), application.rotations(), application.rotationStatus()); } private LockedApplication(Lock lock, ApplicationId id, Instant createdAt, @@ -78,7 +79,7 @@ public class LockedApplication { Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - Optional rotation, Map rotationStatus) { + List rotations, Map rotationStatus) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -93,7 +94,7 @@ public class LockedApplication { this.majorVersion = majorVersion; this.metrics = metrics; this.pemDeployKey = pemDeployKey; - this.rotation = rotation; + this.rotations = rotations; this.rotationStatus = rotationStatus; } @@ -101,35 +102,35 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.with(issueId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, @@ -137,14 +138,14 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withCompletion(projectId, jobType, completion, jobError), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotation, rotationStatus); + pemDeployKey, rotations, rotationStatus); } public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withTriggering(jobType, job), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -195,45 +196,45 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, Optional.ofNullable(issueId), owner, - majorVersion, metrics, pemDeployKey, rotation, rotationStatus); + majorVersion, metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, Optional.ofNullable(owner), majorVersion, metrics, pemDeployKey, - rotation, rotationStatus); + rotations, rotationStatus); } /** Set a major version for this, or set to null to remove any major version override */ @@ -241,31 +242,31 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication with(MetricsService.ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withPemDeployKey(String pemDeployKey) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, Optional.ofNullable(pemDeployKey), rotation, rotationStatus); + metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus); } public LockedApplication with(RotationId rotation) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, Optional.of(rotation), rotationStatus); + metrics, pemDeployKey, List.of(rotation), rotationStatus); } public LockedApplication withRotationStatus(Map rotationStatus) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } /** Don't expose non-leaf sub-objects. */ @@ -278,7 +279,7 @@ public class LockedApplication { private LockedApplication with(Map deployments) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotation, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 3c2cbade606..a2754a57546 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -70,7 +70,8 @@ public class ApplicationSerializer { private final String writeQualityField = "writeQuality"; private final String queryQualityField = "queryQuality"; private final String pemDeployKeyField = "pemDeployKey"; - private final String rotationField = "rotation"; + private final String rotationsField = "rotations"; + private final String deprecatedRotationField = "rotation"; private final String rotationStatusField = "rotationStatus"; // Deployment fields @@ -162,7 +163,8 @@ public class ApplicationSerializer { root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); application.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey)); - application.rotation().ifPresent(rotation -> root.setString(rotationField, rotation.asString())); + Cursor rotations = root.setArray(rotationsField); + application.rotations().forEach(rotation -> rotations.addString(rotation.asString())); toSlime(application.rotationStatus(), root.setArray(rotationStatusField)); return slime; } @@ -329,12 +331,12 @@ public class ApplicationSerializer { ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); Optional pemDeployKey = optionalString(root.field(pemDeployKeyField)); - Optional rotation = rotationFromSlime(root.field(rotationField)); + List rotations = rotationsFromSlime(root); Map rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField)); return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotation, rotationStatus); + pemDeployKey, rotations, rotationStatus); } private List deploymentsFromSlime(Inspector array) { @@ -514,7 +516,25 @@ public class ApplicationSerializer { Instant.ofEpochMilli(object.field(atField).asLong()))); } - private Optional rotationFromSlime(Inspector field) { + private List rotationsFromSlime(Inspector root) { + final var rotations = rotationListFromSlime(root.field(rotationsField)); + final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); + legacyRotation.ifPresent(rotations::add); + return rotations; + } + + private List rotationListFromSlime(Inspector field) { + final var rotations = new ArrayList(); + + for (int i = 0; i < field.entries(); ++i) { + var entry = field.entry(i); + rotations.add(new RotationId(entry.asString())); + } + + return rotations; + } + + private Optional legacyRotationFromSlime(Inspector field) { return field.valid() ? optionalString(field).map(RotationId::new) : Optional.empty(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 872f4d5d1af..d605e7ceacc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -496,7 +496,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .map(URI::toString) .forEach(globalRotationsArray::addString); - application.rotation().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); + + application.rotations().stream().findFirst().ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations Set routingPolicies = controller.applications().routingPolicies(application.id()); @@ -515,7 +516,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { for (Deployment deployment : deployments) { Cursor deploymentObject = instancesArray.addObject(); - if (application.rotation().isPresent() && deployment.zone().environment() == Environment.prod) { + if ((! application.rotations().isEmpty()) && deployment.zone().environment() == Environment.prod) { toSlime(application.rotationStatus(deployment), deploymentObject); } @@ -707,7 +708,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); Application application = controller.applications().require(applicationId); ZoneId zone = ZoneId.from(environment, region); - if (!application.rotation().isPresent()) { + if (application.rotations().isEmpty()) { throw new NotExistsException("global rotation does not exist for " + application); } Deployment deployment = application.deployments().get(zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index b3953c47c01..d2b16721503 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -47,7 +47,7 @@ public class RotationRepository { /** Get rotation for given application */ public Optional getRotation(Application application) { - return application.rotation().map(allRotations::get); + return application.rotations().stream().map(allRotations::get).findFirst(); } /** @@ -60,8 +60,8 @@ public class RotationRepository { * @param lock Lock which must be acquired by the caller */ public Rotation getOrAssignRotation(Application application, RotationLock lock) { - if (application.rotation().isPresent()) { - return allRotations.get(application.rotation().get()); + if (! application.rotations().isEmpty()) { + return allRotations.get(application.rotations().get(0)); } if (application.deploymentSpec().globalServiceId().isEmpty()) { throw new IllegalArgumentException("global-service-id is not set in deployment spec"); @@ -81,8 +81,8 @@ public class RotationRepository { */ public Map availableRotations(@SuppressWarnings("unused") RotationLock lock) { List assignedRotations = applications.asList().stream() - .filter(application -> application.rotation().isPresent()) - .map(application -> application.rotation().get()) + .filter(application -> ! application.rotations().isEmpty()) + .flatMap(application -> application.rotations().stream()) .collect(Collectors.toList()); Map unassignedRotations = new LinkedHashMap<>(this.allRotations); assignedRotations.forEach(unassignedRotations::remove); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 7dcb4aba138..b89941b5d2d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -436,7 +436,7 @@ public class ControllerTest { .build(); tester.deployCompletely(app1, applicationPackage); app1 = tester.applications().require(app1.id()); - assertEquals("rotation-id-02", app1.rotation().get().asString()); + assertEquals("rotation-id-02", app1.rotations().get(0).asString()); // Existing DNS records are updated to point to the newly assigned rotation assertEquals(6, tester.controllerTester().nameService().records().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index d7c0ac9fe9e..7b89b77d105 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -115,7 +115,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), - Optional.of(new RotationId("my-rotation")), + List.of(new RotationId("my-rotation")), rotationStatus); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -151,7 +151,7 @@ public class ApplicationSerializerTest { assertEquals(original.change(), serialized.change()); assertEquals(original.pemDeployKey(), serialized.pemDeployKey()); - assertEquals(original.rotation().get(), serialized.rotation().get()); + assertEquals(original.rotations(), serialized.rotations()); assertEquals(original.rotationStatus(), serialized.rotationStatus()); // Test cluster utilization diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 8d1f40260e3..02a82e35f10 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -14,10 +14,11 @@ import org.junit.Test; import org.junit.rules.ExpectedException; import java.net.URI; +import java.util.List; import java.util.Optional; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; /** * @author Oyvind Gronnesby @@ -64,7 +65,7 @@ public class RotationRepositoryTest { Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); application = tester.applications().require(application.id()); - assertEquals(expected.id(), application.rotation().get()); + assertEquals(List.of(expected.id()), application.rotations()); assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), application.endpointsIn(SystemName.main).main().get().url()); try (RotationLock lock = repository.lock()) { @@ -80,7 +81,7 @@ public class RotationRepositoryTest { .searchDefinition("search foo { }") // Update application package so there is something to deploy .build(); tester.deployCompletely(application, applicationPackage, 43); - assertEquals(expected.id(), tester.applications().require(application.id()).rotation().get()); + assertEquals(List.of(expected.id()), tester.applications().require(application.id()).rotations()); } @Test @@ -139,8 +140,7 @@ public class RotationRepositoryTest { .build(); tester.deployCompletely(application, applicationPackage); Application app = tester.applications().require(application.id()); - Optional rotation = app.rotation(); - assertFalse(rotation.isPresent()); + assertTrue(app.rotations().isEmpty()); } @Test @@ -153,7 +153,7 @@ public class RotationRepositoryTest { Application application = tester.createApplication("app2", "tenant2", 22L, 2L); tester.deployCompletely(application, applicationPackage); - assertEquals(new RotationId("foo-1"), tester.applications().require(application.id()).rotation().get()); + assertEquals(List.of(new RotationId("foo-1")), tester.applications().require(application.id()).rotations()); assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", tester.applications().require(application.id()) .endpointsIn(SystemName.cd).main().get().url().toString()); } -- cgit v1.2.3 From 32cd406c396738486a43e4603f6e07c1561766ab Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 28 May 2019 09:30:57 +0200 Subject: Only update routing policies for applicable zones --- .../src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java | 3 +++ .../main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java | 5 +++++ .../yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java | 2 ++ 3 files changed, 10 insertions(+) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java index ab44fdcb05e..46efe7a440d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilter.java @@ -19,6 +19,9 @@ public interface ZoneFilter { /** Zones which are upgraded by the controller. */ ZoneList controllerUpgraded(); + /** Zones which support direct routing through exclusive load balancers. */ + ZoneList directlyRouted(); + /** Zones where config servers are up and running. */ ZoneList reachable(); diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java index 675b770dc29..528b1974f7d 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java @@ -47,6 +47,11 @@ public class ZoneFilterMock implements ZoneList { return all(); } + @Override + public ZoneList directlyRouted() { + return all(); + } + @Override public ZoneList reachable() { return all(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java index 07cbb34d378..508401b0e14 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java @@ -70,6 +70,8 @@ public class RoutingPolicies { * load balancers for given application have changed. */ public void refresh(ApplicationId application, ZoneId zone) { + // TODO: Use this to decide how apply routing policies for shared routing layer + if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; var lbs = new LoadBalancers(application, zone, controller.applications().configServer() .getLoadBalancers(application, zone)); removeObsoleteEndpointsFromDns(lbs); -- cgit v1.2.3 From 9181c57ff455965293ec75b03d7b29e188ed9577 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 28 May 2019 14:26:14 +0200 Subject: Add missing space in log message --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 7a42456f2ac..0134e4411db 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -519,7 +519,7 @@ public class InternalStepRunner implements StepRunner { }); } catch (IllegalStateException e) { - logger.log(INFO, "Job '" + id.type() + "'no longer supposed to run?:", e); + logger.log(INFO, "Job '" + id.type() + "' no longer supposed to run?", e); return Optional.of(error); } return Optional.of(running); -- cgit v1.2.3 From 24b9d8b7a33773026cb4bb8bb3d6924d9644abd4 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 28 May 2019 14:26:35 +0200 Subject: Skip writing queue when unchanged --- .../yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java | 1 + 1 file changed, 1 insertion(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java index 8878ac9bd5b..d4dc068c71f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/NameServiceDispatcher.java @@ -41,6 +41,7 @@ public class NameServiceDispatcher extends Maintainer { try (Lock lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); NameServiceQueue remaining = queue.dispatchTo(nameService, requestCount); + if (queue == remaining) return; // Queue unchanged db.writeNameServiceQueue(remaining); } } -- cgit v1.2.3 From 1f6db8cf97e7c00a200fc9abe4891429ea441223 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 28 May 2019 14:34:28 +0200 Subject: Remove unused method --- .../yahoo/vespa/hosted/controller/dns/CreateRecord.java | 5 ----- .../yahoo/vespa/hosted/controller/dns/CreateRecords.java | 5 ----- .../vespa/hosted/controller/dns/NameServiceForwarder.java | 15 +++++++-------- .../vespa/hosted/controller/dns/NameServiceQueue.java | 6 ++++-- .../vespa/hosted/controller/dns/NameServiceRequest.java | 6 ------ .../yahoo/vespa/hosted/controller/dns/RemoveRecords.java | 5 ----- 6 files changed, 11 insertions(+), 31 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java index b17ca52d835..3cd3d969731 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecord.java @@ -42,11 +42,6 @@ public class CreateRecord implements NameServiceRequest { } } - @Override - public List affectedRecords() { - return List.of(record); - } - @Override public String toString() { return "create record " + record; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java index ec943676962..9dd02735638 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/CreateRecords.java @@ -51,11 +51,6 @@ public class CreateRecords implements NameServiceRequest { } } - @Override - public List affectedRecords() { - return records(); - } - @Override public String toString() { return "create records " + records(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index f4bea8b1083..299ea168c7a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -39,24 +39,24 @@ public class NameServiceForwarder { } /** Create or update a CNAME record with given name and data */ - public Record createCname(RecordName name, RecordData canonicalName, NameServiceQueue.Priority priority) { - return forward(new CreateRecord(new Record(Record.Type.CNAME, name, canonicalName)), priority).affectedRecords().get(0); + public void createCname(RecordName name, RecordData canonicalName, NameServiceQueue.Priority priority) { + forward(new CreateRecord(new Record(Record.Type.CNAME, name, canonicalName)), priority); } /** Create or update an ALIAS record with given name and targets */ - public List createAlias(RecordName name, Set targets, NameServiceQueue.Priority priority) { + public void createAlias(RecordName name, Set targets, NameServiceQueue.Priority priority) { var records = targets.stream().map(AliasTarget::asData) .map(data -> new Record(Record.Type.ALIAS, name, data)) .collect(Collectors.toList()); - return forward(new CreateRecords(records), priority).affectedRecords(); + forward(new CreateRecords(records), priority); } /** Create or update a TXT record with given name and data */ - public List createTxt(RecordName name, List txtData, NameServiceQueue.Priority priority) { + public void createTxt(RecordName name, List txtData, NameServiceQueue.Priority priority) { var records = txtData.stream() .map(data -> new Record(Record.Type.TXT, name, data)) .collect(Collectors.toList()); - return forward(new CreateRecords(records), priority).affectedRecords(); + forward(new CreateRecords(records), priority); } /** Remove all records of given type and name */ @@ -69,7 +69,7 @@ public class NameServiceForwarder { forward(new RemoveRecords(type, data), priority); } - private NameServiceRequest forward(NameServiceRequest request, NameServiceQueue.Priority priority) { + private void forward(NameServiceRequest request, NameServiceQueue.Priority priority) { try (Lock lock = db.lockNameServiceQueue()) { NameServiceQueue queue = db.readNameServiceQueue(); var queued = queue.requests().size(); @@ -80,7 +80,6 @@ public class NameServiceForwarder { } db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests)); } - return request; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index cc49e589cbb..684fb091d92 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -72,7 +72,7 @@ public class NameServiceQueue { if (requests.isEmpty()) return this; var queue = new NameServiceQueue(requests); - for (var i = 0; i < n && !queue.requests.isEmpty(); i++) { + for (int i = 0; i < n && !queue.requests.isEmpty(); i++) { var request = queue.requests.peek(); try { request.dispatchTo(nameService); @@ -97,11 +97,13 @@ public class NameServiceQueue { /** Priority of a request added to this */ public enum Priority { + /** Default priority. Request will be delivered in FIFO order */ normal, - /**Request is queued before others. Useful for code that needs to act on effects of a request */ + /** Request is queued first. Useful for code that needs to act on effects of a request */ high + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java index a01719ccc88..65076694160 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceRequest.java @@ -2,9 +2,6 @@ package com.yahoo.vespa.hosted.controller.dns; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; - -import java.util.List; /** * Interface for requests to a {@link NameService}. @@ -16,7 +13,4 @@ public interface NameServiceRequest { /** Send this to given name service */ void dispatchTo(NameService nameService); - /** Returns the records affected by executing this */ - List affectedRecords(); - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java index b721f66e452..ddc4d157afd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java @@ -60,11 +60,6 @@ public class RemoveRecords implements NameServiceRequest { nameService.removeRecords(records); } - @Override - public List affectedRecords() { - return List.of(); - } - @Override public String toString() { return "remove records of type " + type + ", by " + -- cgit v1.2.3 From c27d7f4211e6d2357f649323b0c0dba8a7c79ad6 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 31 May 2019 09:53:23 +0200 Subject: Make rotations list in Application immutable --- .../src/main/java/com/yahoo/vespa/hosted/controller/Application.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 1bae9e0ccd4..d989a9dfbe6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -99,7 +99,7 @@ public class Application { this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.pemDeployKey = pemDeployKey; - this.rotations = Objects.requireNonNull(rotations, "rotations cannot be null"); + this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); } @@ -197,7 +197,7 @@ public class Application { /** Returns the global rotation id of this, if present */ public List rotations() { - return Collections.unmodifiableList(rotations); + return rotations; } /** Returns the default global endpoints for this in given system */ -- cgit v1.2.3 From f0c78fa139c408cf59052cc83933215416d67d3e Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 31 May 2019 09:53:43 +0200 Subject: Use ArrayTraverser to build the list of rotations --- .../hosted/controller/persistence/ApplicationSerializer.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index a2754a57546..f45cb5831ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -526,10 +526,10 @@ public class ApplicationSerializer { private List rotationListFromSlime(Inspector field) { final var rotations = new ArrayList(); - for (int i = 0; i < field.entries(); ++i) { - var entry = field.entry(i); - rotations.add(new RotationId(entry.asString())); - } + field.traverse((ArrayTraverser) (idx, inspector) -> { + final var rotation = new RotationId(inspector.asString()); + rotations.add(rotation); + }); return rotations; } -- cgit v1.2.3 From 50e94e6fd2b8940054ec94779225c89eeeafd53d Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 31 May 2019 11:22:21 +0200 Subject: Add test for handling both rotation and rotations --- .../persistence/ApplicationSerializerTest.java | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 7b89b77d105..1269db0d42c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import org.junit.Test; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -244,4 +245,32 @@ public class ApplicationSerializerTest { // ok if no error } + /** TODO: Test can be removed after June 2019 - once legacy field for single rotation is retired */ + @Test + public void testParsingLegacyRotationElement() throws IOException { + // Use the 'complete-application.json' as a baseline + final var applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); + final var slime = SlimeUtils.jsonToSlime(applicationJson); + + // Add the necessary fields to the Slime representation of the application + final var cursor = slime.get(); + + cursor.setString("rotation", "single-rotation"); + + final var rotations = cursor.setArray("rotations"); + rotations.addString("multiple-rotation-1"); + rotations.addString("multiple-rotation-2"); + + // Parse and test the output from parsing contains both legacy rotation and multiple rotations + final var application = applicationSerializer.fromSlime(slime); + + assertEquals( + List.of( + new RotationId("multiple-rotation-1"), + new RotationId("multiple-rotation-2"), + new RotationId("single-rotation") + ), + application.rotations() + ); + } } -- cgit v1.2.3 From c048b608a8a5d1ef593e85c7821336027cd7c39b Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 31 May 2019 11:23:23 +0200 Subject: Add TODO for removing legacy format for rotation --- .../yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java | 1 + 1 file changed, 1 insertion(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index f45cb5831ab..92fec897914 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -534,6 +534,7 @@ public class ApplicationSerializer { return rotations; } + // TODO: Remove after June 2019 once the 'rotation' field is gone from storage private Optional legacyRotationFromSlime(Inspector field) { return field.valid() ? optionalString(field).map(RotationId::new) : Optional.empty(); } -- cgit v1.2.3 From 22817df6fbb31c52e8183e1ccec6e0ee71d05984 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 31 May 2019 11:29:26 +0200 Subject: Change persisted field from rotations => endpoints --- .../vespa/hosted/controller/persistence/ApplicationSerializer.java | 2 +- .../vespa/hosted/controller/persistence/ApplicationSerializerTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 92fec897914..3dd091919aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -70,7 +70,7 @@ public class ApplicationSerializer { private final String writeQualityField = "writeQuality"; private final String queryQualityField = "queryQuality"; private final String pemDeployKeyField = "pemDeployKey"; - private final String rotationsField = "rotations"; + private final String rotationsField = "endpoints"; private final String deprecatedRotationField = "rotation"; private final String rotationStatusField = "rotationStatus"; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 1269db0d42c..084ac593f4c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -257,7 +257,7 @@ public class ApplicationSerializerTest { cursor.setString("rotation", "single-rotation"); - final var rotations = cursor.setArray("rotations"); + final var rotations = cursor.setArray("endpoints"); rotations.addString("multiple-rotation-1"); rotations.addString("multiple-rotation-2"); -- cgit v1.2.3 From 70c1f28c6c16f41191fa736157973b32b7397f64 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 31 May 2019 13:51:21 +0200 Subject: Audit log PUT requests --- .../vespa/hosted/controller/auditlog/AuditLog.java | 1 + .../controller/persistence/AuditLogSerializer.java | 2 + .../controller/auditlog/AuditLoggerTest.java | 46 ++++++++++++++++------ 3 files changed, 38 insertions(+), 11 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java index c467a4a0acd..aefe8ae7b48 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLog.java @@ -109,6 +109,7 @@ public class AuditLog { public enum Method { POST, PATCH, + PUT, DELETE } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index 5bcb155efcb..7fee9a7f9b4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -60,6 +60,7 @@ public class AuditLogSerializer { switch (method) { case POST: return "POST"; case PATCH: return "PATCH"; + case PUT: return "PUT"; case DELETE: return "DELETE"; default: throw new IllegalArgumentException("No serialization defined for method " + method); } @@ -69,6 +70,7 @@ public class AuditLogSerializer { switch (field.asString()) { case "POST": return AuditLog.Entry.Method.POST; case "PATCH": return AuditLog.Entry.Method.PATCH; + case "PUT": return AuditLog.Entry.Method.PUT; case "DELETE": return AuditLog.Entry.Method.DELETE; default: throw new IllegalArgumentException("Unknown serialized value '" + field.asString() + "'"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java index 6470ce3663f..67979571f73 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/auditlog/AuditLoggerTest.java @@ -24,11 +24,10 @@ import static org.junit.Assert.assertTrue; public class AuditLoggerTest { private final ControllerTester tester = new ControllerTester(); + private final Supplier log = () -> tester.controller().auditLogger().readLog(); @Test public void test_logging() { - Supplier log = () -> tester.controller().auditLogger().readLog(); - { // GET request is ignored HttpRequest request = testRequest(Method.GET, URI.create("http://localhost:8080/os/v1/"), ""); tester.controller().auditLogger().log(request); @@ -40,11 +39,8 @@ public class AuditLoggerTest { String data = "{\"cloud\":\"cloud9\",\"version\":\"42.0\"}"; HttpRequest request = testRequest(Method.PATCH, url, data); tester.controller().auditLogger().log(request); - - assertEquals(instant(), log.get().entries().get(0).at()); + assertEntry(Entry.Method.PATCH, 1, "/os/v1/?foo=bar"); assertEquals("user", log.get().entries().get(0).principal()); - assertEquals(Entry.Method.PATCH, log.get().entries().get(0).method()); - assertEquals("/os/v1/?foo=bar", log.get().entries().get(0).resource()); assertEquals(data, log.get().entries().get(0).data().get()); } @@ -53,9 +49,31 @@ public class AuditLoggerTest { HttpRequest request = testRequest(Method.PATCH, URI.create("http://localhost:8080/os/v1/"), "{\"cloud\":\"cloud9\",\"version\":\"43.0\"}"); tester.controller().auditLogger().log(request); - assertEquals(2, log.get().entries().size()); - assertEquals(instant(), log.get().entries().get(0).at()); - assertEquals("/os/v1/", log.get().entries().get(0).resource()); + assertEntry(Entry.Method.PATCH, 2, "/os/v1/"); + } + + { // PUT is logged + tester.clock().advance(Duration.ofDays(1)); + HttpRequest request = testRequest(Method.PUT, URI.create("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1/"), + ""); + tester.controller().auditLogger().log(request); + assertEntry(Entry.Method.PUT, 3, "/zone/v2/prod/us-north-1/nodes/v2/state/dirty/node1/"); + } + + { // DELETE is logged + tester.clock().advance(Duration.ofDays(1)); + HttpRequest request = testRequest(Method.DELETE, URI.create("http://localhost:8080/zone/v2/prod/us-north-1/nodes/v2/node/node1"), + ""); + tester.controller().auditLogger().log(request); + assertEntry(Entry.Method.DELETE, 4, "/zone/v2/prod/us-north-1/nodes/v2/node/node1"); + } + + { // POST is logged + tester.clock().advance(Duration.ofDays(1)); + HttpRequest request = testRequest(Method.POST, URI.create("http://localhost:8080/controller/v1/jobs/upgrader/confidence/6.42"), + "6.42"); + tester.controller().auditLogger().log(request); + assertEntry(Entry.Method.POST, 5, "/controller/v1/jobs/upgrader/confidence/6.42"); } { // 14 days pass and another PATCH request is logged. Older entries are removed due to expiry @@ -63,8 +81,7 @@ public class AuditLoggerTest { HttpRequest request = testRequest(Method.PATCH, URI.create("http://localhost:8080/os/v1/"), "{\"cloud\":\"cloud9\",\"version\":\"44.0\"}"); tester.controller().auditLogger().log(request); - assertEquals(1, log.get().entries().size()); - assertEquals(instant(), log.get().entries().get(0).at()); + assertEntry(Entry.Method.PATCH, 1, "/os/v1/"); } } @@ -72,6 +89,13 @@ public class AuditLoggerTest { return tester.clock().instant().truncatedTo(MILLIS); } + private void assertEntry(Entry.Method method, int logSize, String resource) { + assertEquals(logSize, log.get().entries().size()); + assertEquals(instant(), log.get().entries().get(0).at()); + assertEquals(method, log.get().entries().get(0).method()); + assertEquals(resource, log.get().entries().get(0).resource()); + } + private static HttpRequest testRequest(Method method, URI url, String data) { HttpRequest request = HttpRequest.createTestRequest( url.toString(), -- cgit v1.2.3 From fd9b1c8a4b7b90b5883c636d9dd0c040a3cbe779 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Fri, 31 May 2019 14:13:51 +0200 Subject: Run DeploymentMetricsMaintainer only for non-public systems Add allOf method to SystemName taking a Predicate as argument --- config-provisioning/abi-spec.json | 3 ++- .../src/main/java/com/yahoo/config/provision/SystemName.java | 7 +++++++ .../src/test/java/com/yahoo/config/provision/SystemNameTest.java | 8 ++++++++ .../controller/maintenance/DeploymentMetricsMaintainer.java | 5 ++++- 4 files changed, 21 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 66cc491d471..3cfa301894a 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -783,7 +783,8 @@ "public java.lang.String value()", "public boolean isPublic()", "public boolean isCd()", - "public static java.util.Set all()" + "public static java.util.Set all()", + "public static java.util.Set allOf(java.util.function.Predicate)" ], "fields": [ "public static final enum com.yahoo.config.provision.SystemName cd", diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 088d0bdb3c3..ba462b9eb64 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -3,6 +3,9 @@ package com.yahoo.config.provision; import java.util.EnumSet; import java.util.Set; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * Systems in hosted Vespa @@ -72,4 +75,8 @@ public enum SystemName { public boolean isCd() { return isCd; } public static Set all() { return EnumSet.allOf(SystemName.class); } + + public static Set allOf(Predicate predicate) { + return Stream.of(values()).filter(predicate::test).collect(Collectors.toSet()); + } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java index 6ffca4918a8..eb066958254 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java @@ -3,6 +3,8 @@ package com.yahoo.config.provision; import org.junit.Test; +import java.util.Set; + import static org.junit.Assert.assertEquals; /** @@ -15,4 +17,10 @@ public class SystemNameTest { assertEquals(name, SystemName.from(name.value())); } } + + @Test + public void allOf() { + assertEquals(Set.of(SystemName.cd, SystemName.PublicCd, SystemName.vaas), SystemName.allOf(SystemName::isCd)); + assertEquals(Set.of(SystemName.PublicCd, SystemName.Public, SystemName.vaas), SystemName.allOf(SystemName::isPublic)); + } } \ No newline at end of file diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java index 0cf89d798a7..4ad5940f8f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DeploymentMetricsMaintainer.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.Controller; @@ -20,6 +21,7 @@ import java.util.concurrent.ForkJoinPool; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; @@ -39,7 +41,8 @@ public class DeploymentMetricsMaintainer extends Maintainer { private final ApplicationController applications; public DeploymentMetricsMaintainer(Controller controller, Duration duration, JobControl jobControl) { - super(controller, duration, jobControl); + super(controller, duration, jobControl, DeploymentMetricsMaintainer.class.getSimpleName(), + SystemName.allOf(Predicate.not(SystemName::isPublic))); this.applications = controller.applications(); } -- cgit v1.2.3 From 5482c9e52aaac5d8e2b473218bfcc967954d977b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 31 May 2019 14:26:41 +0200 Subject: Supply Vespa version to config convergence --- .../controller/api/integration/configserver/ConfigServer.java | 8 +++++++- .../hosted/controller/application/SystemApplication.java | 6 ++++-- .../hosted/controller/deployment/InternalStepRunner.java | 11 ++++++----- .../vespa/hosted/controller/maintenance/SystemUpgrader.java | 2 +- .../yahoo/vespa/hosted/controller/versions/VersionStatus.java | 2 +- .../vespa/hosted/controller/integration/ConfigServerMock.java | 2 +- 6 files changed, 20 insertions(+), 11 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 43ae29ee922..03b3d586b73 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.configserver; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; @@ -72,7 +73,12 @@ public interface ConfigServer { NodeRepository nodeRepository(); /** Get service convergence status for given deployment */ - Optional serviceConvergence(DeploymentId deployment); + default Optional serviceConvergence(DeploymentId deployment) { + return serviceConvergence(deployment, Optional.empty()); + } + + /** Get service convergence status for given deployment, using the nodes in the model at the given Vespa version. */ + Optional serviceConvergence(DeploymentId deployment, Optional version); /** Get all load balancers in given zone */ List getLoadBalancers(ZoneId zone); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 26233c998f8..3eeaf09c10b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.application; +import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; @@ -9,6 +10,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -61,11 +63,11 @@ public enum SystemApplication { } /** Returns whether config for this application has converged in given zone */ - public boolean configConvergedIn(ZoneId zone, Controller controller) { + public boolean configConvergedIn(ZoneId zone, Controller controller, Optional version) { if (!hasApplicationPackage()) { return true; } - return controller.configServer().serviceConvergence(new DeploymentId(id(), zone)) + return controller.configServer().serviceConvergence(new DeploymentId(id(), zone), version) .map(ServiceConvergence::converged) .orElse(false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 0134e4411db..68e84c7d289 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -268,7 +268,7 @@ public class InternalStepRunner implements StepRunner { logger.log("Checking installation of " + platform + " and " + application.id() + " ..."); if ( nodesConverged(id.application(), id.type(), platform, logger) - && servicesConverged(id.application(), id.type(), logger)) { + && servicesConverged(id.application(), id.type(), platform, logger)) { if (endpointsAvailable(id.application(), id.type().zone(controller.system()), logger)) { logger.log("Installation succeeded!"); return Optional.of(running); @@ -298,7 +298,7 @@ public class InternalStepRunner implements StepRunner { Version platform = controller.jobController().run(id).get().versions().targetPlatform(); logger.log("Checking installation of tester container ..."); if ( nodesConverged(id.tester().id(), id.type(), platform, logger) - && servicesConverged(id.tester().id(), id.type(), logger)) { + && servicesConverged(id.tester().id(), id.type(), platform, logger)) { if (endpointsAvailable(id.tester().id(), id.type().zone(controller.system()), logger)) { logger.log("Tester container successfully installed!"); return Optional.of(running); @@ -354,9 +354,10 @@ public class InternalStepRunner implements StepRunner { && node.rebootGeneration() >= node.wantedRebootGeneration()); } - private boolean servicesConverged(ApplicationId id, JobType type, DualLogger logger) { - Optional convergence = controller.configServer().serviceConvergence(new DeploymentId(id, type.zone(controller.system()))); - if ( ! convergence.isPresent()) { + private boolean servicesConverged(ApplicationId id, JobType type, Version platform, DualLogger logger) { + var convergence = controller.configServer().serviceConvergence(new DeploymentId(id, type.zone(controller.system())), + Optional.of(platform)); + if (convergence.isEmpty()) { logger.log("Config status not currently available -- will retry."); return false; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 62d401cf478..3c2455b314a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -42,7 +42,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { return minVersion(zone, application, Node::currentVersion).map(target::equals) .orElse(true) - && application.configConvergedIn(zone, controller()); + && application.configConvergedIn(zone, controller(), Optional.of(target)); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index e2d4c90f443..879575669cd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -157,7 +157,7 @@ public class VersionStatus { ListMap versions = new ListMap<>(); for (ZoneId zone : zones) { for (SystemApplication application : SystemApplication.all()) { - boolean configConverged = application.configConvergedIn(zone, controller); + boolean configConverged = application.configConvergedIn(zone, controller, Optional.empty()); if (!configConverged) { log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone + " has not converged"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 913c9a800c1..31b48bea883 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -195,7 +195,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public Optional serviceConvergence(DeploymentId deployment) { + public Optional serviceConvergence(DeploymentId deployment, Optional version) { return Optional.ofNullable(serviceStatus.get(deployment)); } -- cgit v1.2.3 From 5b29a06b30010499b50113f1d656cb97a7de7062 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 31 May 2019 14:48:54 +0200 Subject: Use SystemName#isCd and SystemName#isPublic --- .../yahoo/vespa/hosted/controller/application/Endpoint.java | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 1b90a43421b..6c9b8dd0784 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -119,7 +119,7 @@ public class Endpoint { private static String separator(SystemName system, boolean directRouting, boolean tls) { if (!tls) return "."; if (directRouting) return "."; - if (isPublic(system)) return "."; + if (system.isPublic()) return "."; return "--"; } @@ -141,7 +141,7 @@ public class Endpoint { } private static String systemPart(SystemName system, String separator) { - if (system == SystemName.main || isPublic(system)) return ""; + if (!system.isCd()) return ""; return system.value() + separator; } @@ -160,10 +160,6 @@ public class Endpoint { } } - private static boolean isPublic(SystemName system) { // TODO: Remove and inline once we're down to one - return system == SystemName.Public || system == SystemName.vaas; - } - /** An endpoint's scope */ public enum Scope { @@ -277,7 +273,7 @@ public class Endpoint { } else { throw new IllegalArgumentException("Must set either cluster or rotation target"); } - if (isPublic(system) && !directRouting) { + if (system.isPublic() && !directRouting) { throw new IllegalArgumentException("Public system only supports direct routing endpoints"); } if (directRouting && !port.isDefault()) { -- cgit v1.2.3 From d38e103c001104e5c5e180dff8006d7484d70a41 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Sat, 1 Jun 2019 19:09:05 +0200 Subject: Upgrade tenant host application in SystemUpgrader --- .../hosted/controller/ApplicationController.java | 4 +- .../controller/application/SystemApplication.java | 28 ++++------ .../hosted/controller/maintenance/OsUpgrader.java | 7 +-- .../controller/versions/OsVersionStatus.java | 4 +- .../controller/integration/ConfigServerMock.java | 18 ++---- .../controller/maintenance/OsUpgraderTest.java | 25 ++++----- .../controller/maintenance/SystemUpgraderTest.java | 6 +- .../os/responses/versions-all-upgraded.json | 65 ++++++++++++++++++---- .../restapi/os/responses/versions-initial.json | 65 ++++++++++++++++++---- .../os/responses/versions-partially-upgraded.json | 63 ++++++++++++++++++--- 10 files changed, 201 insertions(+), 84 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index de5d8de1f9e..a511b33313b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -395,7 +395,7 @@ public class ApplicationController { deploySystemApplicationPackage(application, zone, version); } else { // Deploy by calling node repository directly - application.nodeTypes().forEach(nodeType -> configServer().nodeRepository().upgrade(zone, nodeType, version)); + configServer().nodeRepository().upgrade(zone, application.nodeType(), version); } } @@ -406,7 +406,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 3eeaf09c10b..d31ae185b76 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -11,8 +11,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceCon import java.util.List; import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; /** * This represents a system-level application in hosted Vespa. All infrastructure nodes in a hosted Vespa zones are @@ -25,23 +23,17 @@ public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), Set.of(NodeType.proxy, NodeType.host), + tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host), + zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, configServerHost, proxyHost, configServer); private final ApplicationId id; - private final Set nodeTypes; + private final NodeType nodeType; private final List dependencies; SystemApplication(ApplicationId id, NodeType nodeType, SystemApplication... dependencies) { - this(id, Set.of(nodeType), dependencies); - } - - SystemApplication(ApplicationId id, Set nodeTypes, SystemApplication... dependencies) { - if (nodeTypes.isEmpty()) { - throw new IllegalArgumentException("Node types must be non-empty"); - } this.id = id; - this.nodeTypes = Set.copyOf(nodeTypes); + this.nodeType = nodeType; this.dependencies = List.of(dependencies); } @@ -49,9 +41,9 @@ public enum SystemApplication { return id; } - /** The node type(s) that are implicitly allocated to this */ - public Set nodeTypes() { - return nodeTypes; + /** The node type that is implicitly allocated to this */ + public NodeType nodeType() { + return nodeType; } /** Returns the system applications that should upgrade before this */ @@ -73,8 +65,8 @@ public enum SystemApplication { } /** Returns the node types of this that should receive OS upgrades */ - public Set nodeTypesWithUpgradableOs() { - return nodeTypes().stream().filter(NodeType::isDockerHost).collect(Collectors.toSet()); + public boolean isEligibleForOsUpgrades() { + return nodeType.isDockerHost(); } /** All known system applications */ @@ -84,7 +76,7 @@ public enum SystemApplication { @Override public String toString() { - return String.format("system application %s of type %s", id, nodeTypes); + return String.format("system application %s of type %s", id, nodeType); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 3b521657f15..ed3dd552085 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -39,12 +39,11 @@ public class OsUpgrader extends InfrastructureUpgrader { @Override protected void upgrade(Version target, SystemApplication application, ZoneId zone) { - if (wantedVersion(zone, application, target).equals(target)) { + if (!application.isEligibleForOsUpgrades() || wantedVersion(zone, application, target).equals(target)) { return; } log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone)); - application.nodeTypesWithUpgradableOs().forEach(nodeType -> controller().configServer().nodeRepository() - .upgradeOs(zone, nodeType, target)); + controller().configServer().nodeRepository().upgradeOs(zone, application.nodeType(), target); } @Override @@ -77,7 +76,7 @@ public class OsUpgrader extends InfrastructureUpgrader { /** Returns whether node in application should be upgraded by this */ public static boolean eligibleForUpgrade(Node node, SystemApplication application) { return upgradableNodeStates.contains(node.state()) && - application.nodeTypesWithUpgradableOs().contains(node.type()); + application.isEligibleForOsUpgrades(); } private static String name(CloudName cloud) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index d55855a2f36..f9ce75d2297 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -67,8 +67,8 @@ public class OsVersionStatus { controller.osVersions().forEach(osVersion -> versions.put(osVersion, new ArrayList<>())); for (SystemApplication application : SystemApplication.all()) { - if (application.nodeTypesWithUpgradableOs().isEmpty()) { - continue; // Avoid querying applications that do not contain nodes with upgradable OS + if (!application.isEligibleForOsUpgrades()) { + continue; // Avoid querying applications that are not eligible for OS upgrades } for (ZoneId zone : zonesToUpgrade(controller)) { controller.configServer().nodeRepository().list(zone, application.id()).stream() diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 31b48bea883..d0cf6a9bdf7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -67,7 +67,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Inject public ConfigServerMock(ZoneRegistryMock zoneRegistry) { - bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all(), Optional.empty()); + bootstrap(zoneRegistry.zones().all().ids(), SystemApplication.all()); } /** Sets the ConfigChangeActions that will be returned on next deployment. */ @@ -90,28 +90,22 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } public void bootstrap(List zones, SystemApplication... applications) { - bootstrap(zones, List.of(applications), Optional.empty()); + bootstrap(zones, List.of(applications)); } - public void bootstrap(List zones, List applications, Optional type) { + public void bootstrap(List zones, List applications) { nodeRepository().clear(); - addNodes(zones, applications, type); + addNodes(zones, applications); } - public void addNodes(List zones, List applications, Optional type) { + public void addNodes(List zones, List applications) { for (ZoneId zone : zones) { for (SystemApplication application : applications) { - NodeType nodeType = type.orElseGet(() -> { - // Zone application has two node types. Use proxy - if (application == SystemApplication.zone) return NodeType.proxy; - if (application.nodeTypes().size() != 1) throw new IllegalArgumentException(application + " has several node types. Unable to detect type automatically"); - return application.nodeTypes().iterator().next(); - }); List nodes = IntStream.rangeClosed(1, 3) .mapToObj(i -> new Node( HostName.from("node-" + i + "-" + application.id().application() .value()), - Node.State.active, nodeType, + Node.State.active, application.nodeType(), Optional.of(application.id()), initialVersion, initialVersion diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 7a008d1f478..7cd82d83500 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -22,7 +22,6 @@ import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; -import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -60,17 +59,15 @@ public class OsUpgraderTest { // Bootstrap system tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4, zone5), - singletonList(SystemApplication.zone), - Optional.of(NodeType.host)); + List.of(SystemApplication.tenantHost)); // Add system applications that exist in a real system, but are currently not upgraded tester.configServer().addNodes(List.of(zone1, zone2, zone3, zone4, zone5), - Collections.singletonList(SystemApplication.configServer), - Optional.empty()); + List.of(SystemApplication.configServer)); // Fail a few nodes. Failed nodes should not affect versions - failNodeIn(zone1, SystemApplication.zone); - failNodeIn(zone3, SystemApplication.zone); + failNodeIn(zone1, SystemApplication.tenantHost); + failNodeIn(zone3, SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -82,13 +79,13 @@ public class OsUpgraderTest { // zone 1: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone1); + assertWanted(version1, SystemApplication.tenantHost, zone1); // Other zones remain on previous version (none) assertWanted(Version.emptyVersion, SystemApplication.zone, zone2, zone3, zone4); // zone 1: completes upgrade - completeUpgrade(version1, SystemApplication.zone, zone1); + completeUpgrade(version1, SystemApplication.tenantHost, zone1); statusUpdater.maintain(); assertEquals(2, nodesOn(version1).size()); assertEquals(11, nodesOn(Version.emptyVersion).size()); @@ -98,21 +95,21 @@ public class OsUpgraderTest { assertWanted(version1, SystemApplication.zone, zone2, zone3); // zone 4: still on previous version - assertWanted(Version.emptyVersion, SystemApplication.zone, zone4); + assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4); // zone 2 and 3: completes upgrade - completeUpgrade(version1, SystemApplication.zone, zone2, zone3); + completeUpgrade(version1, SystemApplication.tenantHost, zone2, zone3); // zone 4: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone4); + assertWanted(version1, SystemApplication.tenantHost, zone4); // zone 4: completes upgrade - completeUpgrade(version1, SystemApplication.zone, zone4); + completeUpgrade(version1, SystemApplication.tenantHost, zone4); // Next run does nothing as all zones are upgraded osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone1, zone2, zone3, zone4); + assertWanted(version1, SystemApplication.tenantHost, zone1, zone2, zone3, zone4); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() .allMatch(node -> node.version().equals(version1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index d28b36265e3..180dff9feba 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -15,7 +15,6 @@ import org.junit.Test; import java.time.Duration; import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -184,7 +183,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all(), Optional.empty()); + tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -198,7 +197,8 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); List allExceptZone = List.of(SystemApplication.configServerHost, SystemApplication.configServer, - SystemApplication.proxyHost); + SystemApplication.proxyHost, + SystemApplication.tenantHost); completeUpgrade(allExceptZone, version2, zone1); systemUpgrader.maintain(); completeUpgrade(SystemApplication.zone, version2, zone1); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json index e1d81a874dc..17f90259fa8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-all-upgraded.json @@ -5,6 +5,11 @@ "targetVersion": true, "cloud": "cloud1", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -18,7 +23,7 @@ { "hostname": "node-1-configserver-host", "environment": "prod", - "region": "us-east-3" + "region": "us-west-1" }, { "hostname": "node-3-configserver-host", @@ -31,12 +36,12 @@ "region": "us-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", - "region": "us-west-1" + "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-east-3" }, @@ -45,23 +50,48 @@ "environment": "prod", "region": "us-east-3" }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, { "hostname": "node-3-proxy-host", "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-tenant-host", + "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-west-1" } @@ -72,6 +102,11 @@ "targetVersion": true, "cloud": "cloud2", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -83,12 +118,12 @@ "region": "eu-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "eu-west-1" }, @@ -98,7 +133,17 @@ "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json index 9c1625fdcd5..86bc272fcd1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-initial.json @@ -5,6 +5,11 @@ "targetVersion": false, "cloud": "cloud1", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -18,7 +23,7 @@ { "hostname": "node-1-configserver-host", "environment": "prod", - "region": "us-east-3" + "region": "us-west-1" }, { "hostname": "node-3-configserver-host", @@ -31,12 +36,12 @@ "region": "us-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", - "region": "us-west-1" + "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-east-3" }, @@ -45,23 +50,48 @@ "environment": "prod", "region": "us-east-3" }, + { + "hostname": "node-2-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, { "hostname": "node-3-proxy-host", "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-proxy-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-2-tenant-host", + "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-2-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-1-proxy-host", + "hostname": "node-1-tenant-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-west-1" } @@ -72,6 +102,11 @@ "targetVersion": false, "cloud": "cloud2", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -83,12 +118,12 @@ "region": "eu-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "eu-west-1" }, @@ -98,7 +133,17 @@ "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "eu-west-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json index 48a96b70df7..e8007fbf6c5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/responses/versions-partially-upgraded.json @@ -5,6 +5,11 @@ "targetVersion": false, "cloud": "cloud1", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-west-1" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -16,12 +21,12 @@ "region": "us-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "us-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-west-1" }, @@ -31,7 +36,17 @@ "region": "us-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-west-1" } @@ -42,6 +57,11 @@ "targetVersion": true, "cloud": "cloud1", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "us-east-3" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -53,12 +73,12 @@ "region": "us-east-3" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "us-east-3" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "us-east-3" }, @@ -68,7 +88,17 @@ "region": "us-east-3" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "us-east-3" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "us-east-3" } @@ -79,6 +109,11 @@ "targetVersion": false, "cloud": "cloud2", "nodes": [ + { + "hostname": "node-1-configserver-host", + "environment": "prod", + "region": "eu-west-1" + }, { "hostname": "node-3-configserver-host", "environment": "prod", @@ -90,12 +125,12 @@ "region": "eu-west-1" }, { - "hostname": "node-1-configserver-host", + "hostname": "node-2-proxy-host", "environment": "prod", "region": "eu-west-1" }, { - "hostname": "node-2-proxy-host", + "hostname": "node-3-proxy-host", "environment": "prod", "region": "eu-west-1" }, @@ -105,7 +140,17 @@ "region": "eu-west-1" }, { - "hostname": "node-3-proxy-host", + "hostname": "node-2-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-1-tenant-host", + "environment": "prod", + "region": "eu-west-1" + }, + { + "hostname": "node-3-tenant-host", "environment": "prod", "region": "eu-west-1" } -- cgit v1.2.3 From 00108d9e0db941b29d62cff4df468ec70de5dee7 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Sun, 2 Jun 2019 19:11:59 +0200 Subject: Zone app does not depend on config-host --- .../yahoo/vespa/hosted/controller/application/SystemApplication.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index d31ae185b76..96c624dbfad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -24,8 +24,7 @@ public enum SystemApplication { proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, - configServerHost, proxyHost, configServer); + zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, proxyHost, configServer); private final ApplicationId id; private final NodeType nodeType; -- cgit v1.2.3 From b04d168a7dd0cbe5d128deec2671362d2104ae6d Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Sun, 2 Jun 2019 19:26:15 +0200 Subject: Reorder and rename zone-application --- .../controller/application/SystemApplication.java | 8 +-- .../restapi/application/ApplicationApiHandler.java | 6 +- .../controller/maintenance/OsUpgraderTest.java | 4 +- .../controller/maintenance/SystemUpgraderTest.java | 82 +++++++++++----------- 4 files changed, 50 insertions(+), 50 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 96c624dbfad..4f4ec092bc4 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -21,10 +21,10 @@ import java.util.Optional; public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), + configServer(ApplicationId.from("hosted-vespa", "proxy-config-servers", "default"), NodeType.config), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), - configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), - tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host), - zone(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, proxyHost, configServer); + proxy(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, proxyHost, configServer), + tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host); private final ApplicationId id; private final NodeType nodeType; @@ -50,7 +50,7 @@ public enum SystemApplication { /** Returns whether this system application has an application package */ public boolean hasApplicationPackage() { - return this == zone; + return this == proxy; } /** Returns whether config for this application has converged in given zone */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 367c9528e8c..78148bf5428 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -935,11 +935,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Inspector deployOptions = SlimeUtils.jsonToSlime(dataParts.get("deployOptions")).get(); /* - * Special handling of the zone application (the only system application with an application package) + * Special handling of the proxy application (the only system application with an application package) * Setting any other deployOptions here is not supported for now (e.g. specifying version), but * this might be handy later to handle emergency downgrades. */ - boolean isZoneApplication = SystemApplication.zone.id().equals(applicationId); + boolean isZoneApplication = SystemApplication.proxy.id().equals(applicationId); if (isZoneApplication) { // TODO jvenstad: Separate out. // Make it explicit that version is not yet supported here String versionStr = deployOptions.field("vespaVersion").asString(); @@ -957,7 +957,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { throw new IllegalArgumentException("Deployment of system applications is not permitted until system version is determined"); } ActivateResult result = controller.applications() - .deploySystemApplicationPackage(SystemApplication.zone, zone, systemVersion.get().versionNumber()); + .deploySystemApplicationPackage(SystemApplication.proxy, zone, systemVersion.get().versionNumber()); return new SlimeJsonResponse(toSlime(result)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 7cd82d83500..75a637c0268 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -82,7 +82,7 @@ public class OsUpgraderTest { assertWanted(version1, SystemApplication.tenantHost, zone1); // Other zones remain on previous version (none) - assertWanted(Version.emptyVersion, SystemApplication.zone, zone2, zone3, zone4); + assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2, zone3, zone4); // zone 1: completes upgrade completeUpgrade(version1, SystemApplication.tenantHost, zone1); @@ -92,7 +92,7 @@ public class OsUpgraderTest { // zone 2 and 3: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.zone, zone2, zone3); + assertWanted(version1, SystemApplication.proxy, zone2, zone3); // zone 4: still on previous version assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index 180dff9feba..cfc888e76dd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -50,14 +50,14 @@ public class SystemUpgraderTest { Version version1 = Version.fromString("6.5"); // Bootstrap a system without host applications tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.configServer, - SystemApplication.zone); + SystemApplication.proxy); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); - failNodeIn(zone3, SystemApplication.zone); + failNodeIn(zone3, SystemApplication.proxy); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.configServer, version1, zone1, zone2, zone3, zone4); - assertCurrentVersion(SystemApplication.zone, version1, zone1, zone2, zone3, zone4); + assertCurrentVersion(SystemApplication.proxy, version1, zone1, zone2, zone3, zone4); // Controller upgrades Version version2 = Version.fromString("6.6"); @@ -70,72 +70,72 @@ public class SystemUpgraderTest { // Other zones remain on previous version assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4); // Zone application is not upgraded yet - assertWantedVersion(SystemApplication.zone, version1, zone1, zone2, zone3, zone4); + assertWantedVersion(SystemApplication.proxy, version1, zone1, zone2, zone3, zone4); // zone1: zone-config-server upgrades completeUpgrade(SystemApplication.configServer, version2, zone1); - // zone 1: zone-application upgrades + // zone 1: proxy-application upgrades systemUpgrader.maintain(); - assertWantedVersion(SystemApplication.zone, version2, zone1); - completeUpgrade(SystemApplication.zone, version2, zone1); - assertTrue("Deployed zone application", - tester.configServer().application(SystemApplication.zone.id()).isPresent()); + assertWantedVersion(SystemApplication.proxy, version2, zone1); + completeUpgrade(SystemApplication.proxy, version2, zone1); + assertTrue("Deployed proxy application", + tester.configServer().application(SystemApplication.proxy.id()).isPresent()); // zone 2, 3 and 4: still targets old version assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3, zone4); - assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4); + assertWantedVersion(SystemApplication.proxy, version1, zone2, zone3, zone4); // zone 2 and 3: upgrade does not start until zone 1 zone-application config converges systemUpgrader.maintain(); assertWantedVersion(SystemApplication.configServer, version1, zone2, zone3); - convergeServices(SystemApplication.zone, zone1); + convergeServices(SystemApplication.proxy, zone1); // zone 2 and 3: zone-config-server upgrades, first in zone 2, then in zone 3 systemUpgrader.maintain(); assertWantedVersion(SystemApplication.configServer, version2, zone2, zone3); assertWantedVersion(SystemApplication.configServer, version1, zone4); - assertWantedVersion(SystemApplication.zone, version1, zone2, zone3, zone4); + assertWantedVersion(SystemApplication.proxy, version1, zone2, zone3, zone4); completeUpgrade(SystemApplication.configServer, version2, zone2); // zone-application starts upgrading in zone 2, while zone-config-server completes upgrade in zone 3 systemUpgrader.maintain(); - assertWantedVersion(SystemApplication.zone, version2, zone2); - assertWantedVersion(SystemApplication.zone, version1, zone3); + assertWantedVersion(SystemApplication.proxy, version2, zone2); + assertWantedVersion(SystemApplication.proxy, version1, zone3); completeUpgrade(SystemApplication.configServer, version2, zone3); - // zone 2 and 3: zone-application upgrades in parallel + // zone 2 and 3: proxy-application upgrades in parallel systemUpgrader.maintain(); - assertWantedVersion(SystemApplication.zone, version2, zone2, zone3); - completeUpgrade(SystemApplication.zone, version2, zone2, zone3); - convergeServices(SystemApplication.zone, zone2, zone3); + assertWantedVersion(SystemApplication.proxy, version2, zone2, zone3); + completeUpgrade(SystemApplication.proxy, version2, zone2, zone3); + convergeServices(SystemApplication.proxy, zone2, zone3); // zone 4: zone-config-server upgrades systemUpgrader.maintain(); assertWantedVersion(SystemApplication.configServer, version2, zone4); - assertWantedVersion(SystemApplication.zone, version1, zone4); + assertWantedVersion(SystemApplication.proxy, version1, zone4); completeUpgrade(SystemApplication.configServer, version2, zone4); // System version remains unchanged until final application upgrades tester.computeVersionStatus(); assertSystemVersion(version1); - // zone 4: zone-application upgrades + // zone 4: proxy-application upgrades systemUpgrader.maintain(); - assertWantedVersion(SystemApplication.zone, version2, zone4); - completeUpgrade(SystemApplication.zone, version2, zone4); + assertWantedVersion(SystemApplication.proxy, version2, zone4); + completeUpgrade(SystemApplication.proxy, version2, zone4); // zone 4: System version remains unchanged until config converges tester.computeVersionStatus(); assertSystemVersion(version1); - convergeServices(SystemApplication.zone, zone4); + convergeServices(SystemApplication.proxy, zone4); tester.computeVersionStatus(); assertSystemVersion(version2); // Next run does nothing as system is now upgraded systemUpgrader.maintain(); assertWantedVersion(SystemApplication.configServer, version2, zone1, zone2, zone3, zone4); - assertWantedVersion(SystemApplication.zone, version2, zone1, zone2, zone3, zone4); + assertWantedVersion(SystemApplication.proxy, version2, zone1, zone2, zone3, zone4); } @Test @@ -144,7 +144,7 @@ public class SystemUpgraderTest { // Bootstrap system tester.configServer().bootstrap(Collections.singletonList(zone1), SystemApplication.configServer, - SystemApplication.zone); + SystemApplication.proxy); Version version1 = Version.fromString("6.5"); tester.upgradeSystem(version1); @@ -156,9 +156,9 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); completeUpgrade(SystemApplication.configServer, version2, zone1); systemUpgrader.maintain(); - completeUpgrade(SystemApplication.zone, version2, zone1); + completeUpgrade(SystemApplication.proxy, version2, zone1); tester.computeVersionStatus(); - assertSystemVersion(version1); // Unchanged until zone-application converges + assertSystemVersion(version1); // Unchanged until proxy-application converges // Controller upgrades again Version version3 = Version.fromString("6.7"); @@ -166,8 +166,8 @@ public class SystemUpgraderTest { assertSystemVersion(version1); assertControllerVersion(version3); - // zone 1: zone-application converges and system version changes - convergeServices(SystemApplication.zone, zone1); + // zone 1: proxy-application converges and system version changes + convergeServices(SystemApplication.proxy, zone1); tester.computeVersionStatus(); assertSystemVersion(version2); assertControllerVersion(version3); @@ -201,23 +201,23 @@ public class SystemUpgraderTest { SystemApplication.tenantHost); completeUpgrade(allExceptZone, version2, zone1); systemUpgrader.maintain(); - completeUpgrade(SystemApplication.zone, version2, zone1); - convergeServices(SystemApplication.zone, zone1); + completeUpgrade(SystemApplication.proxy, version2, zone1); + convergeServices(SystemApplication.proxy, zone1); assertWantedVersion(SystemApplication.all(), version1, zone2, zone3, zone4); // zone 2 and 3: systemUpgrader.maintain(); completeUpgrade(allExceptZone, version2, zone2, zone3); systemUpgrader.maintain(); - completeUpgrade(SystemApplication.zone, version2, zone2, zone3); - convergeServices(SystemApplication.zone, zone2, zone3); + completeUpgrade(SystemApplication.proxy, version2, zone2, zone3); + convergeServices(SystemApplication.proxy, zone2, zone3); assertWantedVersion(SystemApplication.all(), version1, zone4); // zone 4: systemUpgrader.maintain(); completeUpgrade(allExceptZone, version2, zone4); systemUpgrader.maintain(); - completeUpgrade(SystemApplication.zone, version2, zone4); + completeUpgrade(SystemApplication.proxy, version2, zone4); // All done systemUpgrader.maintain(); @@ -232,7 +232,7 @@ public class SystemUpgraderTest { tester.upgradeSystem(version); systemUpgrader.maintain(); assertWantedVersion(SystemApplication.configServer, version, zone1); - assertWantedVersion(SystemApplication.zone, version, zone1); + assertWantedVersion(SystemApplication.proxy, version, zone1); // Controller is downgraded tester.upgradeController(Version.fromString("6.4")); @@ -240,7 +240,7 @@ public class SystemUpgraderTest { // Wanted version for zone remains unchanged systemUpgrader.maintain(); assertWantedVersion(SystemApplication.configServer, version, zone1); - assertWantedVersion(SystemApplication.zone, version, zone1); + assertWantedVersion(SystemApplication.proxy, version, zone1); } @Test @@ -252,10 +252,10 @@ public class SystemUpgraderTest { tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(List.of(SystemApplication.configServerHost, SystemApplication.proxyHost, - SystemApplication.configServer, SystemApplication.zone), + SystemApplication.configServer, SystemApplication.proxy), version1, zone1); assertCurrentVersion(List.of(SystemApplication.configServerHost, SystemApplication.proxyHost, - SystemApplication.configServer, SystemApplication.zone), + SystemApplication.configServer, SystemApplication.proxy), version1, zone2); // System starts upgrading to next version @@ -266,15 +266,15 @@ public class SystemUpgraderTest { systemUpgrader.maintain(); completeUpgrade(SystemApplication.configServer, version2, zone1); systemUpgrader.maintain(); - completeUpgrade(SystemApplication.zone, version2, zone1); - convergeServices(SystemApplication.zone, zone1); + completeUpgrade(SystemApplication.proxy, version2, zone1); + convergeServices(SystemApplication.proxy, zone1); // Confidence is reduced to broken and next zone is not scheduled for upgrade tester.upgrader().overrideConfidence(version2, VespaVersion.Confidence.broken); tester.computeVersionStatus(); systemUpgrader.maintain(); assertWantedVersion(List.of(SystemApplication.configServerHost, SystemApplication.proxyHost, - SystemApplication.configServer, SystemApplication.zone), version1, zone2); + SystemApplication.configServer, SystemApplication.proxy), version1, zone2); } /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ -- cgit v1.2.3 From 216f0969113250f055a21fce493a7a4dda3f046e Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Mon, 3 Jun 2019 10:41:14 +0200 Subject: Use new jvm element for tester container JDisc memory setting --- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 4 +++- controller-server/src/test/resources/test_runner_services.xml-cd | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 68e84c7d289..891a696be9c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -692,7 +692,9 @@ public class InternalStepRunner implements StepRunner { " \n" + " \n" + "\n" + - " \n" + + " \n" + + " \n" + + " \n" + " \n" + "\n"; diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd index df276f9bbbc..bd49f3be55a 100644 --- a/controller-server/src/test/resources/test_runner_services.xml-cd +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -38,6 +38,8 @@ - + + + -- cgit v1.2.3 From 7ed6eea8e86218398d5d62af4dc35e712f2ecf51 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Mon, 3 Jun 2019 15:45:50 +0200 Subject: Support both old rotation field and new endpoints field Now we support both fields again in both serialization and deserialization to avoid issues with upgrades. --- .../yahoo/vespa/hosted/controller/Application.java | 16 ++++++-- .../vespa/hosted/controller/LockedApplication.java | 46 +++++++++++----------- .../persistence/ApplicationSerializer.java | 10 ++++- .../persistence/ApplicationSerializerTest.java | 28 ++++++++++++- 4 files changed, 71 insertions(+), 29 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index d989a9dfbe6..b4e3b8b1a9a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; +import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; @@ -57,6 +58,7 @@ public class Application { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional pemDeployKey; + private final Optional legacyRotation; private final List rotations; private final Map rotationStatus; @@ -66,7 +68,7 @@ public class Application { new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), - Optional.empty(), Collections.emptyList(), Collections.emptyMap()); + Optional.empty(), Optional.empty(), Collections.emptyList(), Collections.emptyMap()); } /** Used from persistence layer: Do not use */ @@ -74,18 +76,18 @@ public class Application { List deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - List rotations, Map rotationStatus) { + Optional legacyRotation, List rotations, Map rotationStatus) { this(id, createdAt, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())), deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } Application(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - List rotations, Map rotationStatus) { + Optional legacyRotation, List rotations, Map rotationStatus) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -99,6 +101,7 @@ public class Application { this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.pemDeployKey = pemDeployKey; + this.legacyRotation = Objects.requireNonNull(legacyRotation, "legacyRotation cannot be null"); this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); } @@ -196,6 +199,11 @@ public class Application { } /** Returns the global rotation id of this, if present */ + public Optional legacyRotation() { + return legacyRotation; + } + + /** Returns all rotations for this application */ public List rotations() { return rotations; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index e8d69f8a577..5f958b74c39 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -56,6 +56,7 @@ public class LockedApplication { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional pemDeployKey; + private final Optional legacyRotation; private final List rotations; private final Map rotationStatus; @@ -71,7 +72,7 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), - application.pemDeployKey(), application.rotations(), application.rotationStatus()); + application.pemDeployKey(), application.legacyRotation(), application.rotations(), application.rotationStatus()); } private LockedApplication(Lock lock, ApplicationId id, Instant createdAt, @@ -79,7 +80,7 @@ public class LockedApplication { Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - List rotations, Map rotationStatus) { + Optional legacyRotation, List rotations, Map rotationStatus) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -94,6 +95,7 @@ public class LockedApplication { this.majorVersion = majorVersion; this.metrics = metrics; this.pemDeployKey = pemDeployKey; + this.legacyRotation = legacyRotation; this.rotations = rotations; this.rotationStatus = rotationStatus; } @@ -102,35 +104,35 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.with(issueId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, @@ -138,14 +140,14 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withCompletion(projectId, jobType, completion, jobError), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotations, rotationStatus); + pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withTriggering(jobType, job), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -196,45 +198,45 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, Optional.ofNullable(issueId), owner, - majorVersion, metrics, pemDeployKey, rotations, rotationStatus); + majorVersion, metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, Optional.ofNullable(owner), majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + legacyRotation, rotations, rotationStatus); } /** Set a major version for this, or set to null to remove any major version override */ @@ -242,31 +244,31 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication with(MetricsService.ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } public LockedApplication withPemDeployKey(String pemDeployKey) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus); + metrics, Optional.ofNullable(pemDeployKey), legacyRotation, rotations, rotationStatus); } public LockedApplication with(RotationId rotation) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, List.of(rotation), rotationStatus); + metrics, pemDeployKey, Optional.of(rotation), List.of(rotation), rotationStatus); } public LockedApplication withRotationStatus(Map rotationStatus) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } /** Don't expose non-leaf sub-objects. */ @@ -279,7 +281,7 @@ public class LockedApplication { private LockedApplication with(Map deployments) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 3dd091919aa..47c7b54264e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -163,6 +163,7 @@ public class ApplicationSerializer { root.setDouble(queryQualityField, application.metrics().queryServiceQuality()); root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); application.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey)); + application.legacyRotation().ifPresent(rotation -> root.setString(deprecatedRotationField, rotation.asString())); Cursor rotations = root.setArray(rotationsField); application.rotations().forEach(rotation -> rotations.addString(rotation.asString())); toSlime(application.rotationStatus(), root.setArray(rotationStatusField)); @@ -331,12 +332,13 @@ public class ApplicationSerializer { ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); Optional pemDeployKey = optionalString(root.field(pemDeployKeyField)); + Optional legacyRotation = optionalString(root.field(deprecatedRotationField)).map(RotationId::new); List rotations = rotationsFromSlime(root); Map rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField)); return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotations, rotationStatus); + pemDeployKey, legacyRotation, rotations, rotationStatus); } private List deploymentsFromSlime(Inspector array) { @@ -519,7 +521,11 @@ public class ApplicationSerializer { private List rotationsFromSlime(Inspector root) { final var rotations = rotationListFromSlime(root.field(rotationsField)); final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); - legacyRotation.ifPresent(rotations::add); + + if (legacyRotation.isPresent() && ! rotations.contains(legacyRotation.get())) { + rotations.add(legacyRotation.get()); + } + return rotations; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 084ac593f4c..be9624fc693 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -116,6 +116,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), + Optional.of(new RotationId("my-rotation")), List.of(new RotationId("my-rotation")), rotationStatus); @@ -254,7 +255,6 @@ public class ApplicationSerializerTest { // Add the necessary fields to the Slime representation of the application final var cursor = slime.get(); - cursor.setString("rotation", "single-rotation"); final var rotations = cursor.setArray("endpoints"); @@ -272,5 +272,31 @@ public class ApplicationSerializerTest { ), application.rotations() ); + + assertEquals( + Optional.of(new RotationId("single-rotation")), application.legacyRotation() + ); + } + + @Test + public void testParsingOnlyLegacyRotationElement() throws IOException { + // Use the 'complete-application.json' as a baseline + final var applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); + final var slime = SlimeUtils.jsonToSlime(applicationJson); + + // Add the necessary fields to the Slime representation of the application + final var cursor = slime.get(); + + cursor.setString("rotation", "single-rotation"); + + // Parse and test the output from parsing contains both legacy rotation and multiple rotations + final var application = applicationSerializer.fromSlime(slime); + + assertEquals( + List.of( + new RotationId("single-rotation") + ), + application.rotations() + ); } } -- cgit v1.2.3 From 8b97b04ea954ef503c16808062827cb4b232009c Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Sun, 2 Jun 2019 20:17:47 +0200 Subject: Do not upgrade unless not yet converged --- .../hosted/controller/maintenance/InfrastructureUpgrader.java | 7 +++++-- .../vespa/hosted/controller/maintenance/SystemUpgrader.java | 9 ++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 0333711ae39..8818a441fbd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -66,9 +66,12 @@ public abstract class InfrastructureUpgrader extends Maintainer { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { - upgrade(target, application, zone); + boolean currentAppConverged = convergedOn(target, application, zone); + if (!currentAppConverged) { + upgrade(target, application, zone); + } + converged &= currentAppConverged; } - converged &= convergedOn(target, application, zone); } return converged; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 3c2455b314a..8ef353248f8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -40,9 +40,12 @@ public class SystemUpgrader extends InfrastructureUpgrader { @Override protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { - return minVersion(zone, application, Node::currentVersion).map(target::equals) - .orElse(true) - && application.configConvergedIn(zone, controller(), Optional.of(target)); + Optional minVersion = minVersion(zone, application, Node::currentVersion); + // Skip application convergence check if there are no nodes belonging to the application in the zone + if (minVersion.isEmpty()) return true; + + return minVersion.get().equals(target) + && application.configConvergedIn(zone, controller(), Optional.of(target)); } @Override -- cgit v1.2.3 From 00b8cbb74946ef3bb4b336754bf9d6b9380547aa Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 4 Jun 2019 14:44:09 +0200 Subject: Remove DnsMaintainer --- .../maintenance/ControllerMaintenance.java | 3 - .../controller/maintenance/DnsMaintainer.java | 69 ---------- .../controller/maintenance/DnsMaintainerTest.java | 141 --------------------- .../restapi/controller/responses/maintenance.json | 3 - 4 files changed, 216 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 3f5f8273922..3ab9731cb76 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -46,7 +46,6 @@ public class ControllerMaintenance extends AbstractComponent { private final ClusterUtilizationMaintainer clusterUtilizationMaintainer; private final DeploymentMetricsMaintainer deploymentMetricsMaintainer; private final ApplicationOwnershipConfirmer applicationOwnershipConfirmer; - private final DnsMaintainer dnsMaintainer; private final SystemUpgrader systemUpgrader; private final List osUpgraders; private final OsVersionStatusUpdater osVersionStatusUpdater; @@ -78,7 +77,6 @@ public class ControllerMaintenance extends AbstractComponent { clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); - dnsMaintainer = new DnsMaintainer(controller, Duration.ofMinutes(5), jobControl); systemUpgrader = new SystemUpgrader(controller, Duration.ofMinutes(1), jobControl); jobRunner = new JobRunner(controller, Duration.ofMinutes(2), jobControl); osUpgraders = osUpgraders(controller, jobControl); @@ -107,7 +105,6 @@ public class ControllerMaintenance extends AbstractComponent { clusterInfoMaintainer.deconstruct(); deploymentMetricsMaintainer.deconstruct(); applicationOwnershipConfirmer.deconstruct(); - dnsMaintainer.deconstruct(); systemUpgrader.deconstruct(); osUpgraders.forEach(Maintainer::deconstruct); osVersionStatusUpdater.deconstruct(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java deleted file mode 100644 index 7e0032f03c5..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainer.java +++ /dev/null @@ -1,69 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.rotation.Rotation; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * Performs DNS maintenance tasks such as removing DNS aliases for unassigned rotations. - * - * @author mpolden - */ -public class DnsMaintainer extends Maintainer { - - private final AtomicInteger rotationIndex = new AtomicInteger(0); - - public DnsMaintainer(Controller controller, Duration interval, JobControl jobControl) { - super(controller, interval, jobControl); - } - - private RotationRepository rotationRepository() { - return controller().applications().rotationRepository(); - } - - @Override - protected void maintain() { - try (RotationLock lock = rotationRepository().lock()) { - Map unassignedRotations = rotationRepository().availableRotations(lock); - rotationToCheckOf(unassignedRotations.values()).ifPresent(this::removeCname); - } - } - - /** Remove CNAME(s) for unassigned rotation */ - private void removeCname(Rotation rotation) { - // When looking up CNAME by data, the data must be a FQDN - controller().nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordData.fqdn(rotation.name()), Priority.normal); - } - - /** - * Returns the rotation that should be checked in this run. We check only one rotation per run to avoid running into - * rate limits that may be imposed by the {@link NameService} implementation. - */ - private Optional rotationToCheckOf(Collection rotations) { - if (rotations.isEmpty()) return Optional.empty(); - List rotationList = new ArrayList<>(rotations); - int index = rotationIndex.getAndUpdate((i) -> { - if (i < rotationList.size() - 1) { - return ++i; - } - return 0; - }); - return Optional.of(rotationList.get(index)); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java deleted file mode 100644 index 6a3eef5a142..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/DnsMaintainerTest.java +++ /dev/null @@ -1,141 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.ValidationId; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.Application; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.rotation.Rotation; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import org.junit.Before; -import org.junit.Test; - -import java.time.Duration; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; - -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; -import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; - -/** - * @author mpolden - */ -public class DnsMaintainerTest { - - private DeploymentTester tester; - private DnsMaintainer maintainer; - - @Before - public void before() { - tester = new DeploymentTester(); - maintainer = new DnsMaintainer(tester.controller(), Duration.ofHours(12), - new JobControl(new MockCuratorDb())); - } - - @Test - public void removes_record_for_unassigned_rotation() { - Application application = tester.createApplication("app1", "tenant1", 1, 1L); - - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .globalServiceId("foo") - .region("us-west-1") - .region("us-central-1") - .build(); - - Function> findCname = (name) -> tester.controllerTester().nameService() - .findRecords(Record.Type.CNAME, - RecordName.from(name)) - .stream() - .findFirst(); - - // Deploy application - tester.deployCompletely(application, applicationPackage); - assertEquals(3, records().size()); - - Optional record = findCname.apply("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = findCname.apply("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = findCname.apply("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - // DnsMaintainer does nothing - maintainer.maintain(); - assertTrue("DNS record is not removed", findCname.apply("app1--tenant1.global.vespa.yahooapis.com").isPresent()); - assertTrue("DNS record is not removed", findCname.apply("app1--tenant1.global.vespa.oath.cloud").isPresent()); - assertTrue("DNS record is not removed", findCname.apply("app1.tenant1.global.vespa.yahooapis.com").isPresent()); - - // Remove application - applicationPackage = new ApplicationPackageBuilder() - .environment(Environment.prod) - .allow(ValidationId.deploymentRemoval) - .build(); - tester.jobCompletion(component).application(application).nextBuildNumber().uploadArtifact(applicationPackage).submit(); - - tester.deployAndNotify(application, applicationPackage, true, systemTest); - tester.applications().deactivate(application.id(), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); - tester.applications().deactivate(application.id(), ZoneId.from(Environment.staging, RegionName.from("us-east-3"))); - tester.controllerTester().deleteApplication(application.id()); - - // DnsMaintainer removes records - for (int i = 0; i < ControllerTester.availableRotations; i++) { - maintainer.maintain(); - } - tester.flushDnsRequests(); - assertFalse("DNS record removed", findCname.apply("app1--tenant1.global.vespa.yahooapis.com").isPresent()); - assertFalse("DNS record removed", findCname.apply("app1--tenant1.global.vespa.oath.cloud").isPresent()); - assertFalse("DNS record removed", findCname.apply("app1.tenant1.global.vespa.yahooapis.com").isPresent()); - } - - @Test - public void rate_limit_record_removal() { - // Create stale records - int staleTotal = ControllerTester.availableRotations; - for (int i = 1; i <= staleTotal; i++) { - Rotation r = rotation(i); - tester.controllerTester().nameService().createCname(RecordName.from("stale-record-" + i + "." + - Endpoint.OATH_DNS_SUFFIX), - RecordData.from(r.name() + ".")); - } - - // One record is removed per run - for (int i = 1; i <= staleTotal*2; i++) { - maintainer.run(); - tester.flushDnsRequests(); - assertEquals(Math.max(staleTotal - i, 0), records().size()); - } - } - - private Set records() { - return tester.controllerTester().nameService().records(); - } - - private static Rotation rotation(int n) { - String id = String.format("%02d", n); - return new Rotation(new RotationId("rotation-id-" + id), "rotation-fqdn-" + id); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 6218d9d04f0..01b063c84e1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -27,9 +27,6 @@ { "name": "DeploymentMetricsMaintainer" }, - { - "name": "DnsMaintainer" - }, { "name": "JobRunner" }, -- cgit v1.2.3 From a0a508d31f7b726684a1f3ae67204df8a27c0a70 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Tue, 4 Jun 2019 14:44:24 +0200 Subject: Remove DNS records on application deletion only --- .../vespa/hosted/controller/ApplicationController.java | 4 ++++ .../com/yahoo/vespa/hosted/controller/ControllerTest.java | 13 +++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index de5d8de1f9e..e93b7d7245f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -36,6 +36,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; @@ -571,6 +572,9 @@ public class ApplicationController { applicationStore.removeAll(id); applicationStore.removeAll(TesterId.of(id)); + EndpointList endpoints = application.get().endpointsIn(controller.system()); + endpoints.asList().stream().map(Endpoint::dnsName).forEach(name -> controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal)); + log.info("Deleted " + application); })); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b89941b5d2d..b0c11d9738e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -384,16 +384,17 @@ public class ControllerTest { tester.applications().rotationRepository().availableRotations(lock) .containsKey(new RotationId("rotation-id-01"))); } + tester.flushDnsRequests(); - // Records remain + // Records are removed record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); + assertTrue(record.isEmpty()); record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); + assertTrue(record.isEmpty()); record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); + assertTrue(record.isEmpty()); } // Application 2 is deployed and assigned same rotation as application 1 had before deletion @@ -406,7 +407,7 @@ public class ControllerTest { .region("us-central-1") .build(); tester.deployCompletely(app2, applicationPackage); - assertEquals(6, tester.controllerTester().nameService().records().size()); + assertEquals(3, tester.controllerTester().nameService().records().size()); Optional record = tester.controllerTester().findCname("app2--tenant2.global.vespa.yahooapis.com"); assertTrue(record.isPresent()); @@ -438,7 +439,7 @@ public class ControllerTest { app1 = tester.applications().require(app1.id()); assertEquals("rotation-id-02", app1.rotations().get(0).asString()); - // Existing DNS records are updated to point to the newly assigned rotation + // DNS records are created for the newly assigned rotation assertEquals(6, tester.controllerTester().nameService().records().size()); Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); -- cgit v1.2.3 From b078909c47ee13f3a22efd8bab58596ffa9cc724 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 4 Jun 2019 15:35:08 +0200 Subject: proxy -> zone --- .../yahoo/vespa/hosted/controller/application/SystemApplication.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java index 4f4ec092bc4..0d6da51c492 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/SystemApplication.java @@ -21,7 +21,7 @@ import java.util.Optional; public enum SystemApplication { configServerHost(ApplicationId.from("hosted-vespa", "configserver-host", "default"), NodeType.confighost), - configServer(ApplicationId.from("hosted-vespa", "proxy-config-servers", "default"), NodeType.config), + configServer(ApplicationId.from("hosted-vespa", "zone-config-servers", "default"), NodeType.config), proxyHost(ApplicationId.from("hosted-vespa", "proxy-host", "default"), NodeType.proxyhost), proxy(ApplicationId.from("hosted-vespa", "routing", "default"), NodeType.proxy, proxyHost, configServer), tenantHost(ApplicationId.from("hosted-vespa", "tenant-host", "default"), NodeType.host); -- cgit v1.2.3 From 010afac6872cf22b4a0d2160393fc84ce2657764 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 4 Jun 2019 15:57:38 +0200 Subject: State last triggering on unexpected job reports --- .../controller/deployment/DeploymentTrigger.java | 18 ++++++++++-------- .../restapi/application/ApplicationApiTest.java | 9 +++++++-- .../responses/jobreport-unexpected-completion.json | 4 ---- .../jobreport-unexpected-system-test-completion.json | 4 ---- 4 files changed, 17 insertions(+), 18 deletions(-) delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-completion.json delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-system-test-completion.json (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java index d45d5b586fa..b4fe8ea2971 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTrigger.java @@ -124,14 +124,16 @@ public class DeploymentTrigger { } } else { - triggering = application.get().deploymentJobs().statusOf(report.jobType()) - .filter(job -> job.lastTriggered().isPresent() - && job.lastCompleted() - .map(completion -> ! completion.at().isAfter(job.lastTriggered().get().at())) - .orElse(true)) - .orElseThrow(() -> new IllegalStateException("Notified of completion of " + report.jobType().jobName() + " for " + - report.applicationId() + ", but that has neither been triggered nor deployed")) - .lastTriggered().get(); + Optional status = application.get().deploymentJobs().statusOf(report.jobType()); + triggering = status.filter(job -> job.lastTriggered().isPresent() + && job.lastCompleted() + .map(completion -> ! completion.at().isAfter(job.lastTriggered().get().at())) + .orElse(true)) + .orElseThrow(() -> new IllegalStateException("Notified of completion of " + report.jobType().jobName() + " for " + + report.applicationId() + ", but that has not been triggered; last was " + + status.flatMap(job -> job.lastTriggered().map(run -> run.at().toString())) + .orElse("never"))) + .lastTriggered().get(); } application = application.withJobCompletion(report.projectId(), report.jobType(), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index a12a2b5e57a..99bc07a9528 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; +import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -1308,14 +1309,18 @@ public class ApplicationApiTest extends ControllerContainerTest { .data(asJson(job.type(JobType.systemTest).report())) .userIdentity(HOSTED_VESPA_OPERATOR) .get(); - tester.assertResponse(request, new File("jobreport-unexpected-system-test-completion.json"), 400); + tester.assertResponse(request, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Notified of completion " + + "of system-test for tenant1.application1, but that has not been triggered; last was " + + controllerTester.controller().applications().require(app.id()).deploymentJobs().jobStatus().get(JobType.systemTest).lastTriggered().get().at() + "\"}", 400); // Notifying about unknown job fails request = request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) .data(asJson(job.type(JobType.productionUsEast3).report())) .userIdentity(HOSTED_VESPA_OPERATOR) .get(); - tester.assertResponse(request, new File("jobreport-unexpected-completion.json"), 400); + tester.assertResponse(request, "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Notified of completion " + + "of production-us-east-3 for tenant1.application1, but that has not been triggered; last was never\"}", + 400); // ... and assert it was recorded JobStatus recordedStatus = diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-completion.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-completion.json deleted file mode 100644 index 72123e7ae41..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-completion.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "error-code": "BAD_REQUEST", - "message": "Notified of completion of production-us-east-3 for tenant1.application1, but that has neither been triggered nor deployed" -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-system-test-completion.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-system-test-completion.json deleted file mode 100644 index 513cfb754ae..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobreport-unexpected-system-test-completion.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "error-code": "BAD_REQUEST", - "message": "Notified of completion of system-test for tenant1.application1, but that has neither been triggered nor deployed" -} -- cgit v1.2.3 From 39879f9efa836974bd91ef475d5bc73c7553954b Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 4 Jun 2019 17:25:19 +0200 Subject: Do not check config convergence for all applications in VersionStatus unless it has eligible for upgrade nodes --- .../vespa/hosted/controller/versions/VersionStatus.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 879575669cd..5e57e9ebe8e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -6,12 +6,12 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; @@ -157,13 +157,18 @@ public class VersionStatus { ListMap versions = new ListMap<>(); for (ZoneId zone : zones) { for (SystemApplication application : SystemApplication.all()) { + List eligibleForUpgradeApplicationNodes = controller.configServer().nodeRepository() + .list(zone, application.id()).stream() + .filter(SystemUpgrader::eligibleForUpgrade) + .collect(Collectors.toList()); + if (eligibleForUpgradeApplicationNodes.isEmpty()) + continue; + boolean configConverged = application.configConvergedIn(zone, controller, Optional.empty()); if (!configConverged) { log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone + " has not converged"); } - for (Node node : controller.configServer().nodeRepository().list(zone, application.id()).stream() - .filter(SystemUpgrader::eligibleForUpgrade) - .collect(Collectors.toList())) { + for (Node node : eligibleForUpgradeApplicationNodes) { // Only use current node version if config has converged Version nodeVersion = configConverged ? node.currentVersion() : controller.systemVersion(); versions.put(nodeVersion, node.hostname()); -- cgit v1.2.3 From 449935834da39db7e305b365abfd3f8cf1f32721 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 4 Jun 2019 17:25:36 +0200 Subject: Add test to verify configConvergedIn() is not called for inactive sys apps --- .../controller/integration/ConfigServerMock.java | 10 +++++++++- .../controller/maintenance/SystemUpgraderTest.java | 18 ++++++++++++++++-- 2 files changed, 25 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index d0cf6a9bdf7..1894a51adc3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -7,6 +7,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; @@ -21,7 +22,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.serviceview.bindings.ApplicationView; @@ -54,6 +54,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Map endpoints = new HashMap<>(); private final NodeRepositoryMock nodeRepository = new NodeRepositoryMock(); private final Map serviceStatus = new HashMap<>(); + private final Set disallowConvergenceCheckApplications = new HashSet<>(); private final Version initialVersion = new Version(6, 1, 0); private final Set suspendedApplications = new HashSet<>(); private final Map> loadBalancers = new HashMap<>(); @@ -190,9 +191,16 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public Optional serviceConvergence(DeploymentId deployment, Optional version) { + if (disallowConvergenceCheckApplications.contains(deployment.applicationId())) + throw new IllegalStateException(deployment.applicationId() + " should not ask for service convergence"); + return Optional.ofNullable(serviceStatus.get(deployment)); } + public void disallowConvergenceCheck(ApplicationId applicationId) { + disallowConvergenceCheckApplications.add(applicationId); + } + @Override public List getLoadBalancers(ZoneId zone) { return loadBalancers.getOrDefault(zone, Collections.emptyList()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index cfc888e76dd..c8fd7e90ccd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -13,7 +13,6 @@ import org.junit.Before; import org.junit.Test; import java.time.Duration; -import java.util.Collections; import java.util.List; import java.util.function.Function; import java.util.stream.Collectors; @@ -143,7 +142,7 @@ public class SystemUpgraderTest { SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); // Bootstrap system - tester.configServer().bootstrap(Collections.singletonList(zone1), SystemApplication.configServer, + tester.configServer().bootstrap(List.of(zone1), SystemApplication.configServer, SystemApplication.proxy); Version version1 = Version.fromString("6.5"); tester.upgradeSystem(version1); @@ -277,6 +276,21 @@ public class SystemUpgraderTest { SystemApplication.configServer, SystemApplication.proxy), version1, zone2); } + @Test + public void does_not_deploy_proxy_app_in_zones_without_proxy() { + List applications = List.of( + SystemApplication.configServerHost, SystemApplication.configServer, SystemApplication.tenantHost); + tester.configServer().bootstrap(List.of(zone1), applications); + tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id()); + + SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); + + Version version1 = Version.fromString("6.5"); + tester.upgradeSystem(version1); + systemUpgrader.maintain(); + assertCurrentVersion(applications, version1, zone1); + } + /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) { assertWantedVersion(application, version, zones); -- cgit v1.2.3 From ed828e3ff1b706c3809582217c1a889bcca27637 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Wed, 5 Jun 2019 14:09:32 +0200 Subject: Enable maintainer in Main Filter on AWS zones Implemented ofCloud in ZoneFilterMock --- .../config/provision/zone/ZoneFilterMock.java | 6 ++++++ .../maintenance/ResourceMeterMaintainer.java | 4 +++- .../maintenance/CostReportMaintainerTest.java | 12 +++++++++-- .../maintenance/ResourceMeterMaintainerTest.java | 25 ++++++++++++++++------ 4 files changed, 37 insertions(+), 10 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java index 528b1974f7d..1ad4c227d81 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java @@ -1,6 +1,7 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; @@ -77,6 +78,11 @@ public class ZoneFilterMock implements ZoneList { return Collections.unmodifiableList(zones); } + @Override + public ZoneList ofCloud(CloudName cloud) { + return filter(zoneId -> zoneId.cloud().equals(cloud)); + } + private ZoneFilterMock filter(Predicate condition) { return new ZoneFilterMock(zones.stream().filter(negate ? condition.negate() : condition).collect(Collectors.toList()), false); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 5ed7a14836e..9e0db7cc2e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,6 +1,7 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; @@ -44,7 +45,7 @@ public class ResourceMeterMaintainer extends Maintainer { Clock clock, Metric metric, ResourceSnapshotConsumer resourceSnapshotConsumer) { - super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd)); + super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd, SystemName.main)); this.clock = clock; this.nodeRepository = nodeRepository; this.metric = metric; @@ -77,6 +78,7 @@ public class ResourceMeterMaintainer extends Maintainer { private List getNodes() { return controller().zoneRegistry().zones() + .ofCloud(CloudName.from("aws")) .reachable().ids().stream() .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 890af974a0e..5e212a0623d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -1,5 +1,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -10,13 +11,20 @@ import org.junit.Test; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; public class CostReportMaintainerTest { @Test public void maintain() { ControllerTester tester = new ControllerTester(); + List zones = new ArrayList<>(); + zones.add(ZoneId.from("prod", "us-east-3", "yahoo")); + zones.add(ZoneId.from("prod", "us-west-1", "yahoo")); + zones.add(ZoneId.from("prod", "us-central-1", "yahoo")); + zones.add(ZoneId.from("prod", "eu-west-1", "yahoo")); + tester.zoneRegistry().setZones(zones); CostReportConsumer mockConsumer = csv -> Assert.assertEquals(csv, "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" + @@ -42,7 +50,7 @@ public class CostReportMaintainerTest { mockConsumer, new JobControl(tester.curator()), new NodeRepositoryClientMock(), - Clock.fixed(Instant.EPOCH, ZoneId.of("UTC")), + Clock.fixed(Instant.EPOCH, java.time.ZoneId.of("UTC")), costConfig); maintainer.maintain(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index df2a4b5ca7f..9c6ba6aca21 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -1,8 +1,10 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; @@ -10,6 +12,9 @@ import com.yahoo.vespa.hosted.controller.integration.MetricsMock; import org.junit.Test; import java.time.Duration; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; import java.util.Map; import static org.junit.Assert.*; @@ -27,6 +32,12 @@ public class ResourceMeterMaintainerTest { @Test public void testMaintainer() { ControllerTester tester = new ControllerTester(); + List zones = new ArrayList<>(); + zones.add(ZoneId.from("prod", "us-east-3")); + zones.add(ZoneId.from("prod", "us-west-1")); + zones.add(ZoneId.from("prod", "us-central-1")); + zones.add(ZoneId.from("prod", "aws-us-east-1", "aws")); + tester.zoneRegistry().setZones(zones); ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), metrics, snapshotConsumer); resourceMeterMaintainer.maintain(); Map consumedResources = snapshotConsumer.consumedResources(); @@ -36,15 +47,15 @@ public class ResourceMeterMaintainerTest { ResourceSnapshot app1 = consumedResources.get(ApplicationId.from("tenant1", "app1", "default")); ResourceSnapshot app2 = consumedResources.get(ApplicationId.from("tenant2", "app2", "default")); - assertEquals(96, app1.getResourceAllocation().getCpuCores(), DELTA); - assertEquals(96, app1.getResourceAllocation().getMemoryGb(), DELTA); - assertEquals(2000, app1.getResourceAllocation().getDiskGb(), DELTA); + assertEquals(24, app1.getResourceAllocation().getCpuCores(), DELTA); + assertEquals(24, app1.getResourceAllocation().getMemoryGb(), DELTA); + assertEquals(500, app1.getResourceAllocation().getDiskGb(), DELTA); - assertEquals(160, app2.getResourceAllocation().getCpuCores(), DELTA); - assertEquals(96, app2.getResourceAllocation().getMemoryGb(), DELTA); - assertEquals(2000, app2.getResourceAllocation().getDiskGb(), DELTA); + assertEquals(40, app2.getResourceAllocation().getCpuCores(), DELTA); + assertEquals(24, app2.getResourceAllocation().getMemoryGb(), DELTA); + assertEquals(500, app2.getResourceAllocation().getDiskGb(), DELTA); assertEquals(tester.clock().millis()/1000, metrics.getMetric("metering_last_reported")); - assertEquals(4448.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); + assertEquals(1112.0d, (Double) metrics.getMetric("metering_total_reported"), DELTA); } } \ No newline at end of file -- cgit v1.2.3 From c5e32f3da915df1cab7662cf508871e7b341256c Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 6 Jun 2019 09:45:51 +0200 Subject: Handle activation of removed maintainer If an inactive maintainer is removed from the code, the inactive flag is impossible to delete. --- .../restapi/controller/ControllerApiHandler.java | 3 ++- .../restapi/controller/ControllerApiTest.java | 23 ++++++++++++++++++++-- 2 files changed, 23 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java index a59e0e9130f..dde79e78850 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiHandler.java @@ -100,7 +100,8 @@ public class ControllerApiHandler extends AuditLoggingRequestHandler { } private HttpResponse setActive(String jobName, boolean active) { - if ( ! maintenance.jobControl().jobs().contains(jobName)) + boolean activatingInactiveJob = active && !maintenance.jobControl().isActive(jobName); + if (!activatingInactiveJob && !maintenance.jobControl().jobs().contains(jobName)) return ErrorResponse.notFoundError("No job named '" + jobName + "'"); maintenance.jobControl().setActive(jobName, active); return new MessageResponse((active ? "Re-activated" : "Deactivated" ) + " job '" + jobName + "'"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index 8856cc5a97f..ba81c5cf4e4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -17,6 +17,7 @@ import java.io.File; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.time.Instant; +import java.util.Set; import static org.junit.Assert.assertFalse; @@ -40,7 +41,7 @@ public class ControllerApiTest extends ControllerContainerTest { public void testControllerApi() { tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/", new byte[0], Request.Method.GET), new File("root.json")); - // POST deactivation of a maintenance job + // POST deactivates a maintenance job tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", "", Request.Method.POST), "{\"message\":\"Deactivated job 'DeploymentExpirer'\"}", 200); @@ -48,12 +49,30 @@ public class ControllerApiTest extends ControllerContainerTest { // GET a list of all maintenance jobs tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", new byte[0], Request.Method.GET), new File("maintenance.json")); - // DELETE deactivation of a maintenance job + + // DELETE activates maintenance job tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/DeploymentExpirer", "", Request.Method.DELETE), "{\"message\":\"Re-activated job 'DeploymentExpirer'\"}", 200); + // DELETE fails to activate unknown maintenance job + tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/foo", + "", Request.Method.DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No job named 'foo'\"}", + 404); + + // DELETE clears inactive flag for maintenance job that has been removed from the code base + tester.controller().curator().writeInactiveJobs(Set.of("bar")); + tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar", + "", Request.Method.DELETE), + "{\"message\":\"Re-activated job 'bar'\"}", + 200); + tester.assertResponse(hostedOperatorRequest("http://localhost:8080/controller/v1/maintenance/inactive/bar", + "", Request.Method.DELETE), + "{\"error-code\":\"NOT_FOUND\",\"message\":\"No job named 'bar'\"}", + 404); + assertFalse("Actions are logged to audit log", tester.controller().auditLogger().readLog().entries().isEmpty()); } -- cgit v1.2.3 From 37cbca8a4284d623101147942ef39cfa4450db6b Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Thu, 6 Jun 2019 10:21:46 +0200 Subject: Filter on node state --- .../vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 9e0db7cc2e8..a0520f324be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -9,6 +9,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; @@ -82,6 +83,7 @@ public class ResourceMeterMaintainer extends Maintainer { .reachable().ids().stream() .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) + .filter(node -> node.getState() == NodeState.active) .collect(Collectors.toList()); } -- cgit v1.2.3 From 2a08dd6a1b09c441f3af512653285de880116090 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Thu, 6 Jun 2019 11:25:20 +0200 Subject: Repo mock fix --- .../vespa/hosted/controller/integration/NodeRepositoryClientMock.java | 2 ++ 1 file changed, 2 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java index 07978e51263..1c7abd2489d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java @@ -64,6 +64,7 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { membership.clusterid = "clusterA"; membership.clustertype = "container"; node.setMembership(membership); + node.setState(NodeState.active); return node; } @@ -84,6 +85,7 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { membership.clusterid = "clusterB"; membership.clustertype = "content"; node.setMembership(membership); + node.setState(NodeState.active); return node; } -- cgit v1.2.3 From 1046ee18242acf29b71b6e901f7ab4b1e1e5199c Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 6 Jun 2019 00:19:08 +0200 Subject: Move ZoneFilterMock next to ZoneRegistryMock --- .../config/provision/zone/ZoneFilterMock.java | 90 --------------------- .../controller/integration/ZoneFilterMock.java | 93 ++++++++++++++++++++++ .../controller/integration/ZoneRegistryMock.java | 1 - 3 files changed, 93 insertions(+), 91 deletions(-) delete mode 100644 config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java deleted file mode 100644 index 1ad4c227d81..00000000000 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneFilterMock.java +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.config.provision.zone; - -import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.function.Predicate; -import java.util.stream.Collectors; - -/** - * A ZoneList implementation which assumes all zones are controllerManaged. - * - * @author jonmv - */ -public class ZoneFilterMock implements ZoneList { - - private final List zones; - private final boolean negate; - - private ZoneFilterMock(List zones, boolean negate) { - this.negate = negate; - this.zones = zones; - } - - public static ZoneFilter from(Collection zones) { - return new ZoneFilterMock(new ArrayList<>(zones), false); - } - - @Override - public ZoneList not() { - return new ZoneFilterMock(zones, ! negate); - } - - @Override - public ZoneList all() { - return filter(zoneId -> true); - } - - @Override - public ZoneList controllerUpgraded() { - return all(); - } - - @Override - public ZoneList directlyRouted() { - return all(); - } - - @Override - public ZoneList reachable() { - return all(); - } - - @Override - public ZoneList in(Environment... environments) { - return filter(zoneId -> new HashSet<>(Arrays.asList(environments)).contains(zoneId.environment())); - } - - @Override - public ZoneList in(RegionName... regions) { - return filter(zoneId -> new HashSet<>(Arrays.asList(regions)).contains(zoneId.region())); - } - - @Override - public ZoneList among(ZoneId... zones) { - return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId)); - } - - @Override - public List ids() { - return Collections.unmodifiableList(zones); - } - - @Override - public ZoneList ofCloud(CloudName cloud) { - return filter(zoneId -> zoneId.cloud().equals(cloud)); - } - - private ZoneFilterMock filter(Predicate condition) { - return new ZoneFilterMock(zones.stream().filter(negate ? condition.negate() : condition).collect(Collectors.toList()), false); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java new file mode 100644 index 00000000000..e6846144192 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -0,0 +1,93 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.integration; + +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneFilter; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneList; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +/** + * A ZoneList implementation which assumes all zones are controllerManaged. + * + * @author jonmv + */ +public class ZoneFilterMock implements ZoneList { + + private final List zones; + private final boolean negate; + + private ZoneFilterMock(List zones, boolean negate) { + this.negate = negate; + this.zones = zones; + } + + public static ZoneFilter from(Collection zones) { + return new ZoneFilterMock(new ArrayList<>(zones), false); + } + + @Override + public ZoneList not() { + return new ZoneFilterMock(zones, ! negate); + } + + @Override + public ZoneList all() { + return filter(zoneId -> true); + } + + @Override + public ZoneList controllerUpgraded() { + return all(); + } + + @Override + public ZoneList directlyRouted() { + return all(); + } + + @Override + public ZoneList reachable() { + return all(); + } + + @Override + public ZoneList in(Environment... environments) { + return filter(zoneId -> new HashSet<>(Arrays.asList(environments)).contains(zoneId.environment())); + } + + @Override + public ZoneList in(RegionName... regions) { + return filter(zoneId -> new HashSet<>(Arrays.asList(regions)).contains(zoneId.region())); + } + + @Override + public ZoneList among(ZoneId... zones) { + return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId)); + } + + @Override + public List ids() { + return Collections.unmodifiableList(zones); + } + + @Override + public ZoneList ofCloud(CloudName cloud) { + return filter(zoneId -> zoneId.cloud().equals(cloud)); + } + + private ZoneFilterMock filter(Predicate condition) { + return new ZoneFilterMock(zones.stream().filter(negate ? condition.negate() : condition).collect(Collectors.toList()), false); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 12b0f2a930a..23ee245772c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneFilter; -import com.yahoo.config.provision.zone.ZoneFilterMock; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzService; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -- cgit v1.2.3 From 8e87988265e846a6d71c1b29d75a2287a92004da Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 6 Jun 2019 13:48:19 +0200 Subject: Make ZoneList return list of ZoneApi --- .../src/main/java/com/yahoo/config/provision/zone/ZoneList.java | 3 +++ .../yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java | 6 ++++++ 2 files changed, 9 insertions(+) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java index 49c72b9eea8..5f3f2e10898 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java @@ -29,6 +29,9 @@ public interface ZoneList extends ZoneFilter { /** Only the given zones — combine with not() for best effect! */ ZoneList among(ZoneId... zones); + /** Returns the ZoneApi of all zones in this list. */ + List zones(); + /** Returns the id of all zones in this list as — you guessed it — a list. */ List ids(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index e6846144192..059cf92aa52 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.integration; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; @@ -76,6 +77,11 @@ public class ZoneFilterMock implements ZoneList { return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId)); } + @Override + public List zones() { + throw new UnsupportedOperationException(); + } + @Override public List ids() { return Collections.unmodifiableList(zones); -- cgit v1.2.3 From 3780e5ca31f0af3068c354a94181c77234e8c4ed Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 6 Jun 2019 15:34:17 +0200 Subject: Add ZoneApiMock --- .../hosted/controller/integration/ZoneApiMock.java | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java new file mode 100644 index 00000000000..4705982e1f2 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -0,0 +1,70 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.integration; + +import com.yahoo.cloud.config.SentinelConfig; +import com.yahoo.config.model.graph.ModelGraphBuilder; +import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.messagebus.MessagebusConfig; + +/** + * @author hakonhall + */ +public class ZoneApiMock implements ZoneApi { + private final SystemName systemName; + private final ZoneId id; + private final CloudName cloudName; + + public static Builder newBuilder() { return new Builder(); } + + private ZoneApiMock(SystemName systemName, ZoneId id, CloudName cloudName) { + this.systemName = systemName; + this.id = id; + this.cloudName = cloudName; + } + + public static ZoneApiMock fromId(String id) { + return newBuilder().withId(id).build(); + } + + public static ZoneApiMock from(Environment environment, RegionName region) { + return newBuilder().with(ZoneId.from(environment, region)).build(); + } + + @Override + public SystemName getSystemName() { return systemName; } + + @Override + public ZoneId getId() { return id; } + + @Override + public CloudName getCloudName() { return cloudName; } + + public static class Builder { + private SystemName systemName = SystemName.defaultSystem(); + private ZoneId id = ZoneId.defaultId(); + private CloudName cloudName = CloudName.defaultName(); + + public Builder with(ZoneId id) { + this.id = id; + return this; + } + + public Builder withId(String id) { return with(ZoneId.from(id)); } + + public Builder with(CloudName cloudName) { + this.cloudName = cloudName; + return this; + } + + public Builder withCloud(String cloud) { return with(CloudName.from(cloud)); } + + public ZoneApiMock build() { + return new ZoneApiMock(systemName, id, cloudName); + } + } +} -- cgit v1.2.3 From 809e52f09d2298e81c5c88c9d9f8d8456451de44 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 6 Jun 2019 15:52:37 +0200 Subject: Use ZoneApi(Mock) instead of ZoneId in ZoneFilterMock and ZoneRegistryMock --- .../com/yahoo/config/provision/zone/ZoneApi.java | 4 ++ .../com/yahoo/config/provision/zone/ZoneId.java | 4 ++ .../vespa/hosted/controller/ControllerTest.java | 3 +- .../controller/integration/ZoneFilterMock.java | 21 ++++++---- .../controller/integration/ZoneRegistryMock.java | 21 ++++++---- .../controller/maintenance/OsUpgraderTest.java | 49 +++++++++++----------- .../controller/restapi/cost/CostApiTest.java | 9 ++-- .../hosted/controller/restapi/os/OsApiTest.java | 16 +++---- .../controller/restapi/zone/v1/ZoneApiTest.java | 14 ++++--- .../controller/restapi/zone/v2/ZoneApiTest.java | 14 +++---- 10 files changed, 88 insertions(+), 67 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index fd76dc10bdb..9f6ba29d8de 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -17,4 +17,8 @@ public interface ZoneApi { default RegionName getRegionName() { return getId().region(); } CloudName getCloudName(); + + default ZoneId toDeprecatedId() { + return ZoneId.from(getEnvironment(), getRegionName(), getCloudName(), getSystemName()); + } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index 44c6b709cee..c6aaf05492e 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -77,6 +77,10 @@ public class ZoneId { return new ZoneId(Environment.from(environment), RegionName.from(region), CloudName.from(cloud), SystemName.from(system)); } + public static ZoneId defaultId() { + return new ZoneId(Environment.defaultEnvironment(), RegionName.defaultName()); + } + public Environment environment() { return environment; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index b0c11d9738e..3fca94ef21f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -30,6 +30,7 @@ import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.BuildJob; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import org.junit.Test; @@ -462,7 +463,7 @@ public class ControllerTest { Version six = Version.fromString("6.1"); tester.upgradeSystem(six); tester.controllerTester().zoneRegistry().setSystemName(SystemName.cd); - tester.controllerTester().zoneRegistry().setZones(ZoneId.from("prod", "cd-us-central-1")); + tester.controllerTester().zoneRegistry().setZones(ZoneApiMock.fromId("prod.cd-us-central-1")); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .majorVersion(6) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 059cf92aa52..5cf8268c18e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -12,7 +12,6 @@ import com.yahoo.config.provision.zone.ZoneList; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.function.Predicate; @@ -25,15 +24,15 @@ import java.util.stream.Collectors; */ public class ZoneFilterMock implements ZoneList { - private final List zones; + private final List zones; private final boolean negate; - private ZoneFilterMock(List zones, boolean negate) { - this.negate = negate; + private ZoneFilterMock(List zones, boolean negate) { this.zones = zones; + this.negate = negate; } - public static ZoneFilter from(Collection zones) { + public static ZoneFilter from(Collection zones) { return new ZoneFilterMock(new ArrayList<>(zones), false); } @@ -79,12 +78,12 @@ public class ZoneFilterMock implements ZoneList { @Override public List zones() { - throw new UnsupportedOperationException(); + return List.copyOf(zones); } @Override public List ids() { - return Collections.unmodifiableList(zones); + return List.copyOf(zones.stream().map(ZoneApi::toDeprecatedId).collect(Collectors.toList())); } @Override @@ -93,7 +92,13 @@ public class ZoneFilterMock implements ZoneList { } private ZoneFilterMock filter(Predicate condition) { - return new ZoneFilterMock(zones.stream().filter(negate ? condition.negate() : condition).collect(Collectors.toList()), false); + return new ZoneFilterMock( + zones.stream() + .filter(zoneApi -> negate ? + condition.negate().test(zoneApi.toDeprecatedId()) : + condition.test(zoneApi.toDeprecatedId())) + .collect(Collectors.toList()), + false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 23ee245772c..f802a6af549 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneFilter; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzService; @@ -26,6 +27,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.stream.Collectors; /** * @author mpolden @@ -34,7 +36,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry private final Map deploymentTimeToLive = new HashMap<>(); private final Map defaultRegionForEnvironment = new HashMap<>(); - private List zones = new ArrayList<>(); + private List zones = new ArrayList<>(); private SystemName system; private UpgradePolicy upgradePolicy = null; private Map osUpgradePolicies = new HashMap<>(); @@ -50,10 +52,11 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry public ZoneRegistryMock(SystemName system) { this.system = system; - zones.add(ZoneId.from("prod", "us-east-3")); - zones.add(ZoneId.from("prod", "us-west-1")); - zones.add(ZoneId.from("prod", "us-central-1")); - zones.add(ZoneId.from("prod", "eu-west-1")); + setZones(List.of( + ZoneApiMock.fromId("prod.us-east-3"), + ZoneApiMock.fromId("prod.us-west-1"), + ZoneApiMock.fromId("prod.us-central-1"), + ZoneApiMock.fromId("prod.eu-west-1"))); } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { @@ -66,12 +69,12 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry return this; } - public ZoneRegistryMock setZones(List zones) { + public ZoneRegistryMock setZones(List zones) { this.zones = zones; return this; } - public ZoneRegistryMock setZones(ZoneId... zone) { + public ZoneRegistryMock setZones(ZoneApi... zone) { return setZones(List.of(zone)); } @@ -97,7 +100,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public ZoneFilter zones() { - return ZoneFilterMock.from(Collections.unmodifiableList(zones)); + return ZoneFilterMock.from(List.copyOf(zones)); } public AthenzService getConfigServerAthenzIdentity(ZoneId zone) { @@ -146,7 +149,7 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry @Override public boolean hasZone(ZoneId zoneId) { - return zones.contains(zoneId); + return zones.stream().anyMatch(zone -> zone.getId().equals(zoneId)); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 75a637c0268..33bffaae96a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -3,22 +3,21 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; -import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import org.junit.Before; import org.junit.Test; import java.time.Duration; -import java.util.Collections; import java.util.List; -import java.util.Optional; import java.util.function.Function; import java.util.stream.Collectors; @@ -30,11 +29,11 @@ import static org.junit.Assert.assertTrue; */ public class OsUpgraderTest { - private static final ZoneId zone1 = ZoneId.from("prod", "eu-west-1"); - private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1"); - private static final ZoneId zone3 = ZoneId.from("prod", "us-central-1"); - private static final ZoneId zone4 = ZoneId.from("prod", "us-east-3"); - private static final ZoneId zone5 = ZoneId.from("prod", "us-north-1", "other"); + private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").build(); + private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").build(); + private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.us-central-1").build(); + private static final ZoneApi zone4 = ZoneApiMock.newBuilder().withId("prod.us-east-3").build(); + private static final ZoneApi zone5 = ZoneApiMock.newBuilder().withId("prod.us-north-1").withCloud("other").build(); private DeploymentTester tester; private OsVersionStatusUpdater statusUpdater; @@ -50,24 +49,24 @@ public class OsUpgraderTest { public void upgrade_os() { OsUpgrader osUpgrader = osUpgrader( UpgradePolicy.create() - .upgrade(zone1) - .upgradeInParallel(zone2, zone3) - .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader - .upgrade(zone4), + .upgrade(zone1.toDeprecatedId()) + .upgradeInParallel(zone2.toDeprecatedId(), zone3.toDeprecatedId()) + .upgrade(zone5.toDeprecatedId()) // Belongs to a different cloud and is ignored by this upgrader + .upgrade(zone4.toDeprecatedId()), SystemName.cd ); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4, zone5), + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId(), zone5.toDeprecatedId()), List.of(SystemApplication.tenantHost)); // Add system applications that exist in a real system, but are currently not upgraded - tester.configServer().addNodes(List.of(zone1, zone2, zone3, zone4, zone5), + tester.configServer().addNodes(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId(), zone5.toDeprecatedId()), List.of(SystemApplication.configServer)); // Fail a few nodes. Failed nodes should not affect versions - failNodeIn(zone1, SystemApplication.tenantHost); - failNodeIn(zone3, SystemApplication.tenantHost); + failNodeIn(zone1.toDeprecatedId(), SystemApplication.tenantHost); + failNodeIn(zone3.toDeprecatedId(), SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -79,37 +78,37 @@ public class OsUpgraderTest { // zone 1: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone1); + assertWanted(version1, SystemApplication.tenantHost, zone1.toDeprecatedId()); // Other zones remain on previous version (none) - assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2, zone3, zone4); + assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()); // zone 1: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone1); + completeUpgrade(version1, SystemApplication.tenantHost, zone1.toDeprecatedId()); statusUpdater.maintain(); assertEquals(2, nodesOn(version1).size()); assertEquals(11, nodesOn(Version.emptyVersion).size()); // zone 2 and 3: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.proxy, zone2, zone3); + assertWanted(version1, SystemApplication.proxy, zone2.toDeprecatedId(), zone3.toDeprecatedId()); // zone 4: still on previous version - assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4); + assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4.toDeprecatedId()); // zone 2 and 3: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone2, zone3); + completeUpgrade(version1, SystemApplication.tenantHost, zone2.toDeprecatedId(), zone3.toDeprecatedId()); // zone 4: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone4); + assertWanted(version1, SystemApplication.tenantHost, zone4.toDeprecatedId()); // zone 4: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone4); + completeUpgrade(version1, SystemApplication.tenantHost, zone4.toDeprecatedId()); // Next run does nothing as all zones are upgraded osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone1, zone2, zone3, zone4); + assertWanted(version1, SystemApplication.tenantHost, zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() .allMatch(node -> node.version().equals(version1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java index 8cb439fee37..b085a87672e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiTest.java @@ -4,9 +4,10 @@ package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -22,9 +23,9 @@ public class CostApiTest extends ControllerContainerTest { private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser"); private static final CloudName cloud1 = CloudName.from("yahoo"); private static final CloudName cloud2 = CloudName.from("cloud2"); - private static final ZoneId zone1 = ZoneId.from("prod", "us-east-3", cloud1.value()); - private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1", cloud1.value()); - private static final ZoneId zone3 = ZoneId.from("prod", "eu-west-1", cloud2.value()); + private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build(); + private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build(); + private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build(); private ContainerControllerTester tester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index e4b0f526519..f7b5a75fe15 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -5,6 +5,7 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -12,6 +13,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.maintenance.JobControl; import com.yahoo.vespa.hosted.controller.maintenance.Maintainer; @@ -38,9 +40,9 @@ public class OsApiTest extends ControllerContainerTest { private static final AthenzIdentity operator = AthenzUser.fromUserId("operatorUser"); private static final CloudName cloud1 = CloudName.from("cloud1"); private static final CloudName cloud2 = CloudName.from("cloud2"); - private static final ZoneId zone1 = ZoneId.from("prod", "us-east-3", cloud1.value()); - private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1", cloud1.value()); - private static final ZoneId zone3 = ZoneId.from("prod", "eu-west-1", cloud2.value()); + private static final ZoneApi zone1 = ZoneApiMock.newBuilder().withId("prod.us-east-3").with(cloud1).build(); + private static final ZoneApi zone2 = ZoneApiMock.newBuilder().withId("prod.us-west-1").with(cloud1).build(); + private static final ZoneApi zone3 = ZoneApiMock.newBuilder().withId("prod.eu-west-1").with(cloud2).build(); private ContainerControllerTester tester; private List osUpgraders; @@ -51,8 +53,8 @@ public class OsApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(operator); zoneRegistryMock().setSystemName(SystemName.cd) .setZones(zone1, zone2, zone3) - .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2)) - .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3)); + .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1.toDeprecatedId()).upgrade(zone2.toDeprecatedId())) + .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3.toDeprecatedId())); osUpgraders = List.of( new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), @@ -79,12 +81,12 @@ public class OsApiTest extends ControllerContainerTest { // Status is updated after some zones are upgraded upgradeAndUpdateStatus(); - completeUpgrade(zone1); + completeUpgrade(zone1.toDeprecatedId()); assertFile(new Request("http://localhost:8080/os/v1/"), "versions-partially-upgraded.json"); // All zones are upgraded upgradeAndUpdateStatus(); - completeUpgrade(zone2, zone3); + completeUpgrade(zone2.toDeprecatedId(), zone3.toDeprecatedId()); assertFile(new Request("http://localhost:8080/os/v1/"), "versions-all-upgraded.json"); // Downgrade with force is permitted diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java index ca4061f8376..89d90fe6221 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiTest.java @@ -3,12 +3,12 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.api.role.Role; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerCloudTest; -import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; import org.junit.Test; @@ -22,10 +22,12 @@ import java.util.Set; public class ZoneApiTest extends ControllerContainerCloudTest { private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/responses/"; - private static final List zones = List.of(ZoneId.from(Environment.prod, RegionName.from("us-north-1")), - ZoneId.from(Environment.dev, RegionName.from("us-north-2")), - ZoneId.from(Environment.test, RegionName.from("us-north-3")), - ZoneId.from(Environment.staging, RegionName.from("us-north-4"))); + private static final List zones = List.of( + ZoneApiMock.fromId("prod.us-north-1"), + ZoneApiMock.fromId("dev.us-north-2"), + ZoneApiMock.fromId("test.us-north-3"), + ZoneApiMock.fromId("staging.us-north-4")); + private static final Set everyone = Set.of(Role.everyone()); private ContainerControllerTester tester; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java index eff212b6c16..19061b61431 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiTest.java @@ -5,11 +5,12 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Request.Method; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.text.Utf8; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerProxyMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; @@ -29,12 +30,11 @@ public class ZoneApiTest extends ControllerContainerTest { private static final AthenzIdentity HOSTED_VESPA_OPERATOR = AthenzUser.fromUserId("johnoperator"); private static final String responseFiles = "src/test/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/responses/"; - private static final List zones = List.of( - ZoneId.from(Environment.prod, RegionName.from("us-north-1")), - ZoneId.from(Environment.dev, RegionName.from("us-north-2")), - ZoneId.from(Environment.test, RegionName.from("us-north-3")), - ZoneId.from(Environment.staging, RegionName.from("us-north-4")) - ); + private static final List zones = List.of( + ZoneApiMock.fromId("prod.us-north-1"), + ZoneApiMock.fromId("dev.us-north-2"), + ZoneApiMock.fromId("test.us-north-3"), + ZoneApiMock.fromId("staging.us-north-4")); private ContainerControllerTester tester; private ConfigServerProxyMock proxy; -- cgit v1.2.3 From 1c0c87a3366d282aa42940751b17f4a073b821ef Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 6 Jun 2019 16:46:11 +0200 Subject: Use ZoneApi.getCloudName instead of ZoneId.cloud --- .../src/main/java/com/yahoo/vespa/hosted/controller/Controller.java | 5 +++-- .../vespa/hosted/controller/maintenance/ControllerMaintenance.java | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index fde34e62ae4..6f0ee75d098 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -8,6 +8,7 @@ import com.yahoo.component.Vtag; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; @@ -315,8 +316,8 @@ public class Controller extends AbstractComponent { } private Set clouds() { - return zoneRegistry.zones().all().ids().stream() - .map(ZoneId::cloud) + return zoneRegistry.zones().all().zones().stream() + .map(ZoneApi::getCloudName) .collect(Collectors.toUnmodifiableSet()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 3ab9731cb76..c1f896e6593 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.AbstractComponent; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; @@ -117,8 +118,8 @@ public class ControllerMaintenance extends AbstractComponent { /** Create one OS upgrader per cloud found in the zone registry of controller */ private static List osUpgraders(Controller controller, JobControl jobControl) { - return controller.zoneRegistry().zones().controllerUpgraded().ids().stream() - .map(ZoneId::cloud) + return controller.zoneRegistry().zones().controllerUpgraded().zones().stream() + .map(ZoneApi::getCloudName) .distinct() .sorted() .map(cloud -> new OsUpgrader(controller, Duration.ofMinutes(1), jobControl, cloud)) -- cgit v1.2.3 From 9e67220a5caee2b3534814e550b3a4170be9a659 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Thu, 6 Jun 2019 17:59:35 +0200 Subject: Use ZoneApiMock in 2 new tests --- .../controller/maintenance/CostReportMaintainerTest.java | 12 ++++++------ .../controller/maintenance/ResourceMeterMaintainerTest.java | 13 +++++++------ 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java index 5e212a0623d..1edf371924a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; import org.junit.Assert; @@ -19,12 +20,11 @@ public class CostReportMaintainerTest { @Test public void maintain() { ControllerTester tester = new ControllerTester(); - List zones = new ArrayList<>(); - zones.add(ZoneId.from("prod", "us-east-3", "yahoo")); - zones.add(ZoneId.from("prod", "us-west-1", "yahoo")); - zones.add(ZoneId.from("prod", "us-central-1", "yahoo")); - zones.add(ZoneId.from("prod", "eu-west-1", "yahoo")); - tester.zoneRegistry().setZones(zones); + tester.zoneRegistry().setZones( + ZoneApiMock.newBuilder().withId("prod.us-east-3").withCloud("yahoo").build(), + ZoneApiMock.newBuilder().withId("prod.us-west-1").withCloud("yahoo").build(), + ZoneApiMock.newBuilder().withId("prod.us-central-1").withCloud("yahoo").build(), + ZoneApiMock.newBuilder().withId("prod.eu-west-1").withCloud("yahoo").build()); CostReportConsumer mockConsumer = csv -> Assert.assertEquals(csv, "Date,Property,Reserved Cpu Cores,Reserved Memory GB,Reserved Disk Space GB,Usage Fraction\n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java index 9c6ba6aca21..540efcba3b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainerTest.java @@ -9,6 +9,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapsh import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryClientMock; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Test; import java.time.Duration; @@ -32,12 +33,12 @@ public class ResourceMeterMaintainerTest { @Test public void testMaintainer() { ControllerTester tester = new ControllerTester(); - List zones = new ArrayList<>(); - zones.add(ZoneId.from("prod", "us-east-3")); - zones.add(ZoneId.from("prod", "us-west-1")); - zones.add(ZoneId.from("prod", "us-central-1")); - zones.add(ZoneId.from("prod", "aws-us-east-1", "aws")); - tester.zoneRegistry().setZones(zones); + tester.zoneRegistry().setZones( + ZoneApiMock.newBuilder().withId("prod.us-east-3").build(), + ZoneApiMock.newBuilder().withId("prod.us-west-1").build(), + ZoneApiMock.newBuilder().withId("prod.us-central-1").build(), + ZoneApiMock.newBuilder().withId("prod.aws-us-east-1").withCloud("aws").build()); + ResourceMeterMaintainer resourceMeterMaintainer = new ResourceMeterMaintainer(tester.controller(), Duration.ofMinutes(5), new JobControl(tester.curator()), nodeRepository, tester.clock(), metrics, snapshotConsumer); resourceMeterMaintainer.maintain(); Map consumedResources = snapshotConsumer.consumedResources(); -- cgit v1.2.3 From 35e024b57d8e719f627299688a02b446d1b1c1c9 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 15:19:12 +0200 Subject: Only allow specifying ZoneApi to UpgradePolicy, but convert to ZoneId internally --- .../yahoo/config/provision/zone/UpgradePolicy.java | 12 +++-- .../controller/maintenance/OsUpgraderTest.java | 8 +-- .../maintenance/OsVersionStatusUpdaterTest.java | 3 +- .../controller/maintenance/SystemUpgraderTest.java | 62 ++++++++++++---------- .../hosted/controller/restapi/os/OsApiTest.java | 4 +- 5 files changed, 48 insertions(+), 41 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java index d5831efdbaa..54e67ab9940 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java @@ -5,6 +5,8 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.stream.Collectors; +import java.util.stream.Stream; /** * This class declares the order to use when upgrading zones in a system. @@ -20,7 +22,7 @@ public class UpgradePolicy { } public List> asList() { - return Collections.unmodifiableList(zones); + return List.copyOf(zones); } private UpgradePolicy with(ZoneId... zone) { @@ -30,13 +32,13 @@ public class UpgradePolicy { } /** Upgrade given zone as the next step */ - public UpgradePolicy upgrade(ZoneId zone) { - return with(zone); + public UpgradePolicy upgrade(ZoneApi zone) { + return with(zone.toDeprecatedId()); } /** Upgrade given zones in parallel as the next step */ - public UpgradePolicy upgradeInParallel(ZoneId... zone) { - return with(zone); + public UpgradePolicy upgradeInParallel(ZoneApi... zone) { + return with(Stream.of(zone).map(ZoneApi::toDeprecatedId).toArray(ZoneId[]::new)); } public static UpgradePolicy create() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 33bffaae96a..39ff29f4ae0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -49,10 +49,10 @@ public class OsUpgraderTest { public void upgrade_os() { OsUpgrader osUpgrader = osUpgrader( UpgradePolicy.create() - .upgrade(zone1.toDeprecatedId()) - .upgradeInParallel(zone2.toDeprecatedId(), zone3.toDeprecatedId()) - .upgrade(zone5.toDeprecatedId()) // Belongs to a different cloud and is ignored by this upgrader - .upgrade(zone4.toDeprecatedId()), + .upgrade(zone1) + .upgradeInParallel(zone2, zone3) + .upgrade(zone5) // Belongs to a different cloud and is ignored by this upgrader + .upgrade(zone4), SystemName.cd ); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java index d8e6f573592..fe7f39fd66d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneId; @@ -32,7 +33,7 @@ public class OsVersionStatusUpdaterTest { new JobControl(new MockCuratorDb())); // Add all zones to upgrade policy UpgradePolicy upgradePolicy = UpgradePolicy.create(); - for (ZoneId zone : tester.zoneRegistry().zones().controllerUpgraded().ids()) { + for (ZoneApi zone : tester.zoneRegistry().zones().controllerUpgraded().zones()) { upgradePolicy = upgradePolicy.upgrade(zone); } tester.zoneRegistry().setOsUpgradePolicy(CloudName.defaultName(), upgradePolicy); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index c8fd7e90ccd..e68e4171def 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -3,11 +3,13 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.UpgradePolicy; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.NodeRepositoryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import org.junit.Before; import org.junit.Test; @@ -25,10 +27,10 @@ import static org.junit.Assert.assertTrue; */ public class SystemUpgraderTest { - private static final ZoneId zone1 = ZoneId.from("prod", "eu-west-1"); - private static final ZoneId zone2 = ZoneId.from("prod", "us-west-1"); - private static final ZoneId zone3 = ZoneId.from("prod", "us-central-1"); - private static final ZoneId zone4 = ZoneId.from("prod", "us-east-3"); + private static final ZoneApi zone1 = ZoneApiMock.fromId("prod.eu-west-1"); + private static final ZoneApi zone2 = ZoneApiMock.fromId("prod.us-west-1"); + private static final ZoneApi zone3 = ZoneApiMock.fromId("prod.us-central-1"); + private static final ZoneApi zone4 = ZoneApiMock.fromId("prod.us-east-3"); private DeploymentTester tester; @@ -48,8 +50,8 @@ public class SystemUpgraderTest { Version version1 = Version.fromString("6.5"); // Bootstrap a system without host applications - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.configServer, - SystemApplication.proxy); + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), + SystemApplication.configServer, SystemApplication.proxy); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); failNodeIn(zone3, SystemApplication.proxy); @@ -142,7 +144,7 @@ public class SystemUpgraderTest { SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1), SystemApplication.configServer, + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), SystemApplication.configServer, SystemApplication.proxy); Version version1 = Version.fromString("6.5"); tester.upgradeSystem(version1); @@ -182,7 +184,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1, zone2, zone3, zone4), SystemApplication.all()); + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), SystemApplication.all()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -280,7 +282,7 @@ public class SystemUpgraderTest { public void does_not_deploy_proxy_app_in_zones_without_proxy() { List applications = List.of( SystemApplication.configServerHost, SystemApplication.configServer, SystemApplication.tenantHost); - tester.configServer().bootstrap(List.of(zone1), applications); + tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), applications); tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id()); SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); @@ -292,36 +294,38 @@ public class SystemUpgraderTest { } /** Simulate upgrade of nodes allocated to given application. In a real system this is done by the node itself */ - private void completeUpgrade(SystemApplication application, Version version, ZoneId... zones) { + private void completeUpgrade(SystemApplication application, Version version, ZoneApi... zones) { assertWantedVersion(application, version, zones); - for (ZoneId zone : zones) { + for (ZoneApi zone : zones) { for (Node node : listNodes(zone, application)) { - nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), node.owner(), - node.wantedVersion(), node.wantedVersion())); + nodeRepository().putByHostname( + zone.toDeprecatedId(), + new Node(node.hostname(), node.state(), node.type(), node.owner(), node.wantedVersion(), node.wantedVersion())); } assertCurrentVersion(application, version, zone); } } - private void convergeServices(SystemApplication application, ZoneId... zones) { - for (ZoneId zone : zones) { - tester.controllerTester().configServer().convergeServices(application.id(), zone); + private void convergeServices(SystemApplication application, ZoneApi... zones) { + for (ZoneApi zone : zones) { + tester.controllerTester().configServer().convergeServices(application.id(), zone.toDeprecatedId()); } } - private void completeUpgrade(List applications, Version version, ZoneId... zones) { + private void completeUpgrade(List applications, Version version, ZoneApi... zones) { applications.forEach(application -> completeUpgrade(application, version, zones)); } - private void failNodeIn(ZoneId zone, SystemApplication application) { - List nodes = nodeRepository().list(zone, application.id()); + private void failNodeIn(ZoneApi zone, SystemApplication application) { + List nodes = nodeRepository().list(zone.toDeprecatedId(), application.id()); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); - nodeRepository().putByHostname(zone, new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), - node.currentVersion(), node.wantedVersion())); + nodeRepository().putByHostname( + zone.toDeprecatedId(), + new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), node.currentVersion(), node.wantedVersion())); } private void assertSystemVersion(Version version) { @@ -332,33 +336,33 @@ public class SystemUpgraderTest { assertEquals(version, tester.controller().versionStatus().controllerVersion().get().versionNumber()); } - private void assertWantedVersion(SystemApplication application, Version version, ZoneId... zones) { + private void assertWantedVersion(SystemApplication application, Version version, ZoneApi... zones) { assertVersion(application, version, Node::wantedVersion, zones); } - private void assertCurrentVersion(SystemApplication application, Version version, ZoneId... zones) { + private void assertCurrentVersion(SystemApplication application, Version version, ZoneApi... zones) { assertVersion(application, version, Node::currentVersion, zones); } - private void assertWantedVersion(List applications, Version version, ZoneId... zones) { + private void assertWantedVersion(List applications, Version version, ZoneApi... zones) { applications.forEach(application -> assertVersion(application, version, Node::wantedVersion, zones)); } - private void assertCurrentVersion(List applications, Version version, ZoneId... zones) { + private void assertCurrentVersion(List applications, Version version, ZoneApi... zones) { applications.forEach(application -> assertVersion(application, version, Node::currentVersion, zones)); } private void assertVersion(SystemApplication application, Version version, Function versionField, - ZoneId... zones) { - for (ZoneId zone : requireNonEmpty(zones)) { + ZoneApi... zones) { + for (ZoneApi zone : requireNonEmpty(zones)) { for (Node node : listNodes(zone, application)) { assertEquals(application + " version", version, versionField.apply(node)); } } } - private List listNodes(ZoneId zone, SystemApplication application) { - return nodeRepository().list(zone, application.id()).stream() + private List listNodes(ZoneApi zone, SystemApplication application) { + return nodeRepository().list(zone.toDeprecatedId(), application.id()).stream() .filter(SystemUpgrader::eligibleForUpgrade) .collect(Collectors.toList()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index f7b5a75fe15..b2dfd7b4cb6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -53,8 +53,8 @@ public class OsApiTest extends ControllerContainerTest { addUserToHostedOperatorRole(operator); zoneRegistryMock().setSystemName(SystemName.cd) .setZones(zone1, zone2, zone3) - .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1.toDeprecatedId()).upgrade(zone2.toDeprecatedId())) - .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3.toDeprecatedId())); + .setOsUpgradePolicy(cloud1, UpgradePolicy.create().upgrade(zone1).upgrade(zone2)) + .setOsUpgradePolicy(cloud2, UpgradePolicy.create().upgrade(zone3)); osUpgraders = List.of( new OsUpgrader(tester.controller(), Duration.ofDays(1), new JobControl(tester.controller().curator()), -- cgit v1.2.3 From b798ac5044ecb49429ddccb8ab4335de80175f88 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 15:30:18 +0200 Subject: Use ZoneApi internally in UpgradePolicy --- .../yahoo/config/provision/zone/UpgradePolicy.java | 22 +++++++++++++++------- .../maintenance/InfrastructureUpgrader.java | 2 +- .../controller/versions/OsVersionStatus.java | 2 +- 3 files changed, 17 insertions(+), 9 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java index 54e67ab9940..abcb4edce95 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java @@ -15,30 +15,38 @@ import java.util.stream.Stream; */ public class UpgradePolicy { - private final List> zones; + private final List> zones; - private UpgradePolicy(List> zones) { + private UpgradePolicy(List> zones) { this.zones = zones; } - public List> asList() { + public List> deprecatedAsList() { + return zones.stream() + .map(list -> list.stream() + .map(ZoneApi::toDeprecatedId) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + } + + public List> asList() { return List.copyOf(zones); } - private UpgradePolicy with(ZoneId... zone) { - List> zones = new ArrayList<>(this.zones); + private UpgradePolicy with(ZoneApi... zone) { + List> zones = new ArrayList<>(this.zones); zones.add(Arrays.asList(zone)); return new UpgradePolicy(zones); } /** Upgrade given zone as the next step */ public UpgradePolicy upgrade(ZoneApi zone) { - return with(zone.toDeprecatedId()); + return with(zone); } /** Upgrade given zones in parallel as the next step */ public UpgradePolicy upgradeInParallel(ZoneApi... zone) { - return with(Stream.of(zone).map(ZoneApi::toDeprecatedId).toArray(ZoneId[]::new)); + return with(zone); } public static UpgradePolicy create() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 8818a441fbd..c14a454a447 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -42,7 +42,7 @@ public abstract class InfrastructureUpgrader extends Maintainer { /** Deploy a list of system applications until they converge on the given version */ private void upgradeAll(Version target, List applications) { - for (List zones : upgradePolicy.asList()) { + for (List zones : upgradePolicy.deprecatedAsList()) { boolean converged = true; for (ZoneId zone : zones) { try { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index f9ce75d2297..b2c14f96a39 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -87,7 +87,7 @@ public class OsVersionStatus { private static List zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() - .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) + .flatMap(upgradePolicy -> upgradePolicy.deprecatedAsList().stream()) .flatMap(Collection::stream) .collect(Collectors.toUnmodifiableList()); } -- cgit v1.2.3 From a1cd2f4aabfa858f4b394f43891cb1e92593fb50 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 15:46:47 +0200 Subject: Get ZoneApis from UpgradePolicy in upgraders --- .../maintenance/InfrastructureUpgrader.java | 19 ++++++++++--------- .../hosted/controller/maintenance/OsUpgrader.java | 15 ++++++++------- .../hosted/controller/maintenance/SystemUpgrader.java | 11 ++++++----- 3 files changed, 24 insertions(+), 21 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index c14a454a447..7e6ad8cdacb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.config.provision.zone.UpgradePolicy; @@ -42,9 +43,9 @@ public abstract class InfrastructureUpgrader extends Maintainer { /** Deploy a list of system applications until they converge on the given version */ private void upgradeAll(Version target, List applications) { - for (List zones : upgradePolicy.deprecatedAsList()) { + for (List zones : upgradePolicy.asList()) { boolean converged = true; - for (ZoneId zone : zones) { + for (ZoneApi zone : zones) { try { converged &= upgradeAll(target, applications, zone); } catch (UnreachableNodeRepositoryException e) { @@ -62,7 +63,7 @@ public abstract class InfrastructureUpgrader extends Maintainer { } /** Returns whether all applications have converged to the target version in zone */ - private boolean upgradeAll(Version target, List applications, ZoneId zone) { + private boolean upgradeAll(Version target, List applications, ZoneApi zone) { boolean converged = true; for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { @@ -76,28 +77,28 @@ public abstract class InfrastructureUpgrader extends Maintainer { return converged; } - private boolean convergedOn(Version target, List applications, ZoneId zone) { + private boolean convergedOn(Version target, List applications, ZoneApi zone) { return applications.stream().allMatch(application -> convergedOn(target, application, zone)); } /** Upgrade component to target version. Implementation should be idempotent */ - protected abstract void upgrade(Version target, SystemApplication application, ZoneId zone); + protected abstract void upgrade(Version target, SystemApplication application, ZoneApi zone); /** Returns whether application has converged to target version in zone */ - protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneId zone); + protected abstract boolean convergedOn(Version target, SystemApplication application, ZoneApi zone); /** Returns the target version for the component upgraded by this, if any */ protected abstract Optional targetVersion(); /** Returns whether the upgrader should require given node to upgrade */ - protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone); + protected abstract boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone); /** Find the minimum value of a version field in a zone */ - protected final Optional minVersion(ZoneId zone, SystemApplication application, Function versionField) { + protected final Optional minVersion(ZoneApi zone, SystemApplication application, Function versionField) { try { return controller().configServer() .nodeRepository() - .list(zone, application.id()) + .list(zone.toDeprecatedId(), application.id()) .stream() .filter(node -> requireUpgradeOf(node, application, zone)) .map(versionField) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index ed3dd552085..bb6c9065ae0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableSet; import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.config.provision.zone.ZoneId; @@ -38,22 +39,22 @@ public class OsUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneId zone) { + protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { if (!application.isEligibleForOsUpgrades() || wantedVersion(zone, application, target).equals(target)) { return; } log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone)); - controller().configServer().nodeRepository().upgradeOs(zone, application.nodeType(), target); + controller().configServer().nodeRepository().upgradeOs(zone.toDeprecatedId(), application.nodeType(), target); } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { return currentVersion(zone, application, target).equals(target); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) { - return cloud.equals(zone.cloud()) && eligibleForUpgrade(node, application); + protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { + return cloud.equals(zone.getCloudName()) && eligibleForUpgrade(node, application); } @Override @@ -65,11 +66,11 @@ public class OsUpgrader extends InfrastructureUpgrader { .map(OsVersion::version); } - private Version currentVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { + private Version currentVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::currentOsVersion).orElse(defaultVersion); } - private Version wantedVersion(ZoneId zone, SystemApplication application, Version defaultVersion) { + private Version wantedVersion(ZoneApi zone, SystemApplication application, Version defaultVersion) { return minVersion(zone, application, Node::wantedOsVersion).orElse(defaultVersion); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 8ef353248f8..483096bd3b0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -30,26 +31,26 @@ public class SystemUpgrader extends InfrastructureUpgrader { } @Override - protected void upgrade(Version target, SystemApplication application, ZoneId zone) { + protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) .orElse(true)) { log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); - controller().applications().deploy(application, zone, target); + controller().applications().deploy(application, zone.toDeprecatedId(), target); } } @Override - protected boolean convergedOn(Version target, SystemApplication application, ZoneId zone) { + protected boolean convergedOn(Version target, SystemApplication application, ZoneApi zone) { Optional minVersion = minVersion(zone, application, Node::currentVersion); // Skip application convergence check if there are no nodes belonging to the application in the zone if (minVersion.isEmpty()) return true; return minVersion.get().equals(target) - && application.configConvergedIn(zone, controller(), Optional.of(target)); + && application.configConvergedIn(zone.toDeprecatedId(), controller(), Optional.of(target)); } @Override - protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneId zone) { + protected boolean requireUpgradeOf(Node node, SystemApplication application, ZoneApi zone) { return eligibleForUpgrade(node); } -- cgit v1.2.3 From 5a3f260e4e701f8ed0cb654a8efba7c6565f10b0 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 16:01:29 +0200 Subject: Remove ZoneId from UpgradePolicy --- .../java/com/yahoo/config/provision/zone/UpgradePolicy.java | 8 -------- .../vespa/hosted/controller/versions/OsVersionStatus.java | 13 +++++++------ 2 files changed, 7 insertions(+), 14 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java index abcb4edce95..02e57b04791 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java @@ -21,14 +21,6 @@ public class UpgradePolicy { this.zones = zones; } - public List> deprecatedAsList() { - return zones.stream() - .map(list -> list.stream() - .map(ZoneApi::toDeprecatedId) - .collect(Collectors.toList())) - .collect(Collectors.toList()); - } - public List> asList() { return List.copyOf(zones); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index b2c14f96a39..75144bcd880 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -7,6 +7,7 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; @@ -70,12 +71,12 @@ public class OsVersionStatus { if (!application.isEligibleForOsUpgrades()) { continue; // Avoid querying applications that are not eligible for OS upgrades } - for (ZoneId zone : zonesToUpgrade(controller)) { - controller.configServer().nodeRepository().list(zone, application.id()).stream() + for (ZoneApi zone : zonesToUpgrade(controller)) { + controller.configServer().nodeRepository().list(zone.toDeprecatedId(), application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) - .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.environment(), zone.region())) + .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.getEnvironment(), zone.getRegionName())) .forEach(node -> { - var version = new OsVersion(node.version(), zone.cloud()); + var version = new OsVersion(node.version(), zone.getCloudName()); versions.putIfAbsent(version, new ArrayList<>()); versions.get(version).add(node); }); @@ -85,9 +86,9 @@ public class OsVersionStatus { return new OsVersionStatus(versions); } - private static List zonesToUpgrade(Controller controller) { + private static List zonesToUpgrade(Controller controller) { return controller.zoneRegistry().osUpgradePolicies().stream() - .flatMap(upgradePolicy -> upgradePolicy.deprecatedAsList().stream()) + .flatMap(upgradePolicy -> upgradePolicy.asList().stream()) .flatMap(Collection::stream) .collect(Collectors.toUnmodifiableList()); } -- cgit v1.2.3 From 534034b94b8cc1fb055c7ab65d4cbcbe5d05c638 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 16:44:30 +0200 Subject: Be more specific in log messages --- .../java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java | 2 +- .../com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index bb6c9065ae0..4d050173831 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -43,7 +43,7 @@ public class OsUpgrader extends InfrastructureUpgrader { if (!application.isEligibleForOsUpgrades() || wantedVersion(zone, application, target).equals(target)) { return; } - log.info(String.format("Upgrading OS of %s to version %s in %s", application.id(), target, zone)); + log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), target, zone.getId(), zone.getCloudName())); controller().configServer().nodeRepository().upgradeOs(zone.toDeprecatedId(), application.nodeType(), target); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 483096bd3b0..406d37fa924 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -34,7 +34,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { protected void upgrade(Version target, SystemApplication application, ZoneApi zone) { if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) .orElse(true)) { - log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone)); + log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); controller().applications().deploy(application, zone.toDeprecatedId(), target); } } -- cgit v1.2.3 From 75b1acdf58163683101a1923a06bd18cebfd4efb Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 17:21:14 +0200 Subject: Remove usage of ZoneId w/cloud and system --- .../src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java | 2 -- .../java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java | 3 +-- .../com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java | 1 - 3 files changed, 1 insertion(+), 5 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java index 02e57b04791..1baaac772c8 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/UpgradePolicy.java @@ -5,8 +5,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.Stream; /** * This class declares the order to use when upgrading zones in a system. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java index 4d050173831..8845f4c652f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgrader.java @@ -7,7 +7,6 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.versions.OsVersion; @@ -44,7 +43,7 @@ public class OsUpgrader extends InfrastructureUpgrader { return; } log.info(String.format("Upgrading OS of %s to version %s in %s in cloud %s", application.id(), target, zone.getId(), zone.getCloudName())); - controller().configServer().nodeRepository().upgradeOs(zone.toDeprecatedId(), application.nodeType(), target); + controller().configServer().nodeRepository().upgradeOs(zone.getId(), application.nodeType(), target); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 406d37fa924..5a40dd591fd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.application.SystemApplication; -- cgit v1.2.3 From fda39d604e6d06b8f70865c87ad0f8a730f3fbc8 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sat, 8 Jun 2019 17:39:47 +0200 Subject: Replace unnecessary usage of toDeprecatedId --- .../hosted/controller/maintenance/InfrastructureUpgrader.java | 2 +- .../yahoo/vespa/hosted/controller/versions/OsVersionStatus.java | 2 +- .../vespa/hosted/controller/maintenance/SystemUpgraderTest.java | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 7e6ad8cdacb..159eb234aa7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -98,7 +98,7 @@ public abstract class InfrastructureUpgrader extends Maintainer { try { return controller().configServer() .nodeRepository() - .list(zone.toDeprecatedId(), application.id()) + .list(zone.getId(), application.id()) .stream() .filter(node -> requireUpgradeOf(node, application, zone)) .map(versionField) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java index 75144bcd880..f5b9d8263e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/OsVersionStatus.java @@ -72,7 +72,7 @@ public class OsVersionStatus { continue; // Avoid querying applications that are not eligible for OS upgrades } for (ZoneApi zone : zonesToUpgrade(controller)) { - controller.configServer().nodeRepository().list(zone.toDeprecatedId(), application.id()).stream() + controller.configServer().nodeRepository().list(zone.getId(), application.id()).stream() .filter(node -> OsUpgrader.eligibleForUpgrade(node, application)) .map(node -> new Node(node.hostname(), node.currentOsVersion(), zone.getEnvironment(), zone.getRegionName())) .forEach(node -> { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index e68e4171def..cb5e7cc90a1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -299,7 +299,7 @@ public class SystemUpgraderTest { for (ZoneApi zone : zones) { for (Node node : listNodes(zone, application)) { nodeRepository().putByHostname( - zone.toDeprecatedId(), + zone.getId(), new Node(node.hostname(), node.state(), node.type(), node.owner(), node.wantedVersion(), node.wantedVersion())); } @@ -318,13 +318,13 @@ public class SystemUpgraderTest { } private void failNodeIn(ZoneApi zone, SystemApplication application) { - List nodes = nodeRepository().list(zone.toDeprecatedId(), application.id()); + List nodes = nodeRepository().list(zone.getId(), application.id()); if (nodes.isEmpty()) { throw new IllegalArgumentException("No nodes allocated to " + application.id()); } Node node = nodes.get(0); nodeRepository().putByHostname( - zone.toDeprecatedId(), + zone.getId(), new Node(node.hostname(), Node.State.failed, node.type(), node.owner(), node.currentVersion(), node.wantedVersion())); } @@ -362,7 +362,7 @@ public class SystemUpgraderTest { } private List listNodes(ZoneApi zone, SystemApplication application) { - return nodeRepository().list(zone.toDeprecatedId(), application.id()).stream() + return nodeRepository().list(zone.getId(), application.id()).stream() .filter(SystemUpgrader::eligibleForUpgrade) .collect(Collectors.toList()); } -- cgit v1.2.3 From 727e3484393d794a01b72b1a8e5dc92656236e73 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Tue, 11 Jun 2019 11:20:04 +0200 Subject: Support instances --- .../hosted/controller/ApplicationController.java | 2 -- .../controller/deployment/DeploymentTester.java | 6 +++- .../deployment/DeploymentTriggerTest.java | 41 ++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e0a73d994d1..4e6c45998ec 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -228,8 +228,6 @@ public class ApplicationController { * @throws IllegalArgumentException if the application already exists */ public Application createApplication(ApplicationId id, Optional credentials) { - if ( ! (id.instance().isDefault())) // TODO: Support instances properly - throw new IllegalArgumentException("Only the instance name 'default' is supported at the moment"); if (id.instance().isTester()) throw new IllegalArgumentException("'" + id + "' is a tester application!"); try (Lock lock = lock(id)) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index a093aac430b..3ce32347e35 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -156,8 +156,12 @@ public class DeploymentTester { } public Application createApplication(String applicationName, String tenantName, long projectId, long propertyId) { + return createApplication("default", applicationName, tenantName, projectId, propertyId); + } + + public Application createApplication(String instanceName, String applicationName, String tenantName, long projectId, long propertyId) { TenantName tenant = tester.createTenant(tenantName, UUID.randomUUID().toString(), propertyId); - return tester.createApplication(tenant, applicationName, "default", projectId); + return tester.createApplication(tenant, applicationName, instanceName, projectId); } public void restartController() { tester.createNewController(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java index bf7c8cadd3c..0d8d32299ae 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; @@ -108,6 +109,46 @@ public class DeploymentTriggerTest { tester.assertRunning(productionUsWest1, app.id()); } + @Test + public void testIndependentInstances() { + Application instance1 = tester.createApplication("instance1", "app", "tenant", 1, 1L); + Application instance2 = tester.createApplication("instance2", "app", "tenant", 2, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .upgradePolicy("default") + .environment(Environment.prod) + .region("us-west-1") + .build(); + + Version version = Version.fromString("6.2"); + tester.upgradeSystem(version); + + // Deploy completely once + tester.jobCompletion(component).application(instance1).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.productionUsWest1); + + tester.jobCompletion(component).application(instance2).uploadArtifact(applicationPackage).submit(); + tester.deployAndNotify(instance2, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(instance2, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(instance2, applicationPackage, true, JobType.productionUsWest1); + + // New version is released + Version newVersion = Version.fromString("6.3"); + tester.upgradeSystem(newVersion); + + // instance1 upgrades, but not instance 2 + tester.deployAndNotify(instance1, applicationPackage, true, JobType.systemTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.stagingTest); + tester.deployAndNotify(instance1, applicationPackage, true, JobType.productionUsWest1); + + Version instance1Version = tester.application(instance1.id()).deployments().get(JobType.productionUsWest1.zone(main)).version(); + Version instance2Version = tester.application(instance2.id()).deployments().get(JobType.productionUsWest1.zone(main)).version(); + + assertEquals(newVersion, instance1Version); + assertEquals(version, instance2Version); + } + @Test public void abortsInternalJobsOnNewApplicationChange() { InternalDeploymentTester iTester = new InternalDeploymentTester(); -- cgit v1.2.3 From eb3927fff15edaffedf89817378f4b73ce2eef49 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 11 Jun 2019 22:29:28 +0200 Subject: Remove vaas --- config-provisioning/abi-spec.json | 3 +-- .../main/java/com/yahoo/config/provision/SystemName.java | 7 +------ .../java/com/yahoo/config/provision/SystemNameTest.java | 4 ++-- .../controller/api/integration/deployment/JobType.java | 4 ---- .../yahoo/vespa/hosted/controller/api/role/RoleTest.java | 16 ++++++++-------- .../vespa/hosted/controller/application/Endpoint.java | 1 - 6 files changed, 12 insertions(+), 23 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index aa9b196c0e4..18f4d317019 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -790,8 +790,7 @@ "public static final enum com.yahoo.config.provision.SystemName main", "public static final enum com.yahoo.config.provision.SystemName Public", "public static final enum com.yahoo.config.provision.SystemName PublicCd", - "public static final enum com.yahoo.config.provision.SystemName dev", - "public static final enum com.yahoo.config.provision.SystemName vaas" + "public static final enum com.yahoo.config.provision.SystemName dev" ] }, "com.yahoo.config.provision.TenantName": { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index ba462b9eb64..45962fc596a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -27,10 +27,7 @@ public enum SystemName { PublicCd(true, true), /** Local development system */ - dev(false, false), - - /** VaaS */ - vaas(true, true); // TODO: Remove this and use public everywhere + dev(false, false); // TODO: Remove this and use public everywhere private final boolean isPublic; private final boolean isCd; @@ -51,7 +48,6 @@ public enum SystemName { case "main": return main; case "public": return Public; case "publiccd": return PublicCd; - case "vaas": return vaas; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); } } @@ -63,7 +59,6 @@ public enum SystemName { case main: return "main"; case Public: return "public"; case PublicCd: return "publiccd"; - case vaas: return "vaas"; default : throw new IllegalStateException(); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java index eb066958254..f50005edd52 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java @@ -20,7 +20,7 @@ public class SystemNameTest { @Test public void allOf() { - assertEquals(Set.of(SystemName.cd, SystemName.PublicCd, SystemName.vaas), SystemName.allOf(SystemName::isCd)); - assertEquals(Set.of(SystemName.PublicCd, SystemName.Public, SystemName.vaas), SystemName.allOf(SystemName::isPublic)); + assertEquals(Set.of(SystemName.cd, SystemName.PublicCd), SystemName.allOf(SystemName::isCd)); + assertEquals(Set.of(SystemName.PublicCd, SystemName.Public), SystemName.allOf(SystemName::isPublic)); } } \ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 585a8f84fb2..94e111455ac 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -15,7 +15,6 @@ import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.PublicCd; import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; -import static com.yahoo.config.provision.SystemName.vaas; /** Job types that exist in the build system */ public enum JobType { @@ -88,9 +87,6 @@ public enum JobType { devCdUsCentral1 ("dev-cd-us-central-1", Map.of(cd , ZoneId.from("dev" , "cd-us-central-1"))), - devAwsUsEast1b ("dev-aws-us-east-1b", - Map.of(vaas, ZoneId.from("dev" , "vaas-aws-us-east-1b"))), - devAwsUsEast1c ("dev-aws-us-east-1c", Map.of(PublicCd, ZoneId.from("dev", "aws-us-east-1c"))), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 4c11da3b697..d141ef6c73e 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue; public class RoleTest { private static final Enforcer mainEnforcer = new Enforcer(SystemName.main); - private static final Enforcer vaasEnforcer = new Enforcer(SystemName.vaas); + private static final Enforcer publicEnforcer = new Enforcer(SystemName.Public); @Test public void operator_membership() { @@ -40,18 +40,18 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); Role publicSystem = Role.athenzTenantAdmin(TenantName.from("t1")); - assertFalse(vaasEnforcer.allows(publicSystem, Action.read, URI.create("/controller/v1/foo"))); - assertTrue(vaasEnforcer.allows(publicSystem, Action.read, URI.create("/badge/v1/badge"))); - assertTrue(vaasEnforcer.allows(publicSystem, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertFalse(publicEnforcer.allows(publicSystem, Action.read, URI.create("/controller/v1/foo"))); + assertTrue(publicEnforcer.allows(publicSystem, Action.read, URI.create("/badge/v1/badge"))); + assertTrue(publicEnforcer.allows(publicSystem, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); } @Test public void build_service_membership() { Role role = Role.tenantPipeline(TenantName.from("t1"), ApplicationName.from("a1")); - assertFalse(vaasEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); - assertFalse(vaasEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); - assertTrue(vaasEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport"))); - assertFalse("No global read access", vaasEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); + assertFalse(publicEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); + assertFalse(publicEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport"))); + assertFalse("No global read access", publicEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); } @Test diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 6c9b8dd0784..5026ca75a83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -152,7 +152,6 @@ public class Endpoint { if (legacy) return YAHOO_DNS_SUFFIX; return OATH_DNS_SUFFIX; case Public: - case vaas: return PUBLIC_DNS_SUFFIX; case PublicCd: return PUBLIC_CD_DNS_SUFFIX; -- cgit v1.2.3 From a7a5e809c4261e34b8545ee0e73ed96e609d1411 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 12 Jun 2019 07:50:21 +0200 Subject: Revert "Remove vaas" --- config-provisioning/abi-spec.json | 3 ++- .../main/java/com/yahoo/config/provision/SystemName.java | 7 ++++++- .../java/com/yahoo/config/provision/SystemNameTest.java | 4 ++-- .../controller/api/integration/deployment/JobType.java | 4 ++++ .../yahoo/vespa/hosted/controller/api/role/RoleTest.java | 16 ++++++++-------- .../vespa/hosted/controller/application/Endpoint.java | 1 + .../java/ai/vespa/hosted/api/ControllerHttpClient.java | 3 ++- 7 files changed, 25 insertions(+), 13 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index 18f4d317019..aa9b196c0e4 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -790,7 +790,8 @@ "public static final enum com.yahoo.config.provision.SystemName main", "public static final enum com.yahoo.config.provision.SystemName Public", "public static final enum com.yahoo.config.provision.SystemName PublicCd", - "public static final enum com.yahoo.config.provision.SystemName dev" + "public static final enum com.yahoo.config.provision.SystemName dev", + "public static final enum com.yahoo.config.provision.SystemName vaas" ] }, "com.yahoo.config.provision.TenantName": { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 45962fc596a..ba462b9eb64 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -27,7 +27,10 @@ public enum SystemName { PublicCd(true, true), /** Local development system */ - dev(false, false); // TODO: Remove this and use public everywhere + dev(false, false), + + /** VaaS */ + vaas(true, true); // TODO: Remove this and use public everywhere private final boolean isPublic; private final boolean isCd; @@ -48,6 +51,7 @@ public enum SystemName { case "main": return main; case "public": return Public; case "publiccd": return PublicCd; + case "vaas": return vaas; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); } } @@ -59,6 +63,7 @@ public enum SystemName { case main: return "main"; case Public: return "public"; case PublicCd: return "publiccd"; + case vaas: return "vaas"; default : throw new IllegalStateException(); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java index f50005edd52..eb066958254 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java @@ -20,7 +20,7 @@ public class SystemNameTest { @Test public void allOf() { - assertEquals(Set.of(SystemName.cd, SystemName.PublicCd), SystemName.allOf(SystemName::isCd)); - assertEquals(Set.of(SystemName.PublicCd, SystemName.Public), SystemName.allOf(SystemName::isPublic)); + assertEquals(Set.of(SystemName.cd, SystemName.PublicCd, SystemName.vaas), SystemName.allOf(SystemName::isCd)); + assertEquals(Set.of(SystemName.PublicCd, SystemName.Public, SystemName.vaas), SystemName.allOf(SystemName::isPublic)); } } \ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 94e111455ac..585a8f84fb2 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -15,6 +15,7 @@ import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.PublicCd; import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; +import static com.yahoo.config.provision.SystemName.vaas; /** Job types that exist in the build system */ public enum JobType { @@ -87,6 +88,9 @@ public enum JobType { devCdUsCentral1 ("dev-cd-us-central-1", Map.of(cd , ZoneId.from("dev" , "cd-us-central-1"))), + devAwsUsEast1b ("dev-aws-us-east-1b", + Map.of(vaas, ZoneId.from("dev" , "vaas-aws-us-east-1b"))), + devAwsUsEast1c ("dev-aws-us-east-1c", Map.of(PublicCd, ZoneId.from("dev", "aws-us-east-1c"))), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index d141ef6c73e..4c11da3b697 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue; public class RoleTest { private static final Enforcer mainEnforcer = new Enforcer(SystemName.main); - private static final Enforcer publicEnforcer = new Enforcer(SystemName.Public); + private static final Enforcer vaasEnforcer = new Enforcer(SystemName.vaas); @Test public void operator_membership() { @@ -40,18 +40,18 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); Role publicSystem = Role.athenzTenantAdmin(TenantName.from("t1")); - assertFalse(publicEnforcer.allows(publicSystem, Action.read, URI.create("/controller/v1/foo"))); - assertTrue(publicEnforcer.allows(publicSystem, Action.read, URI.create("/badge/v1/badge"))); - assertTrue(publicEnforcer.allows(publicSystem, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertFalse(vaasEnforcer.allows(publicSystem, Action.read, URI.create("/controller/v1/foo"))); + assertTrue(vaasEnforcer.allows(publicSystem, Action.read, URI.create("/badge/v1/badge"))); + assertTrue(vaasEnforcer.allows(publicSystem, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); } @Test public void build_service_membership() { Role role = Role.tenantPipeline(TenantName.from("t1"), ApplicationName.from("a1")); - assertFalse(publicEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); - assertFalse(publicEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); - assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport"))); - assertFalse("No global read access", publicEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); + assertFalse(vaasEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); + assertFalse(vaasEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertTrue(vaasEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport"))); + assertFalse("No global read access", vaasEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); } @Test diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 5026ca75a83..6c9b8dd0784 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -152,6 +152,7 @@ public class Endpoint { if (legacy) return YAHOO_DNS_SUFFIX; return OATH_DNS_SUFFIX; case Public: + case vaas: return PUBLIC_DNS_SUFFIX; case PublicCd: return PUBLIC_CD_DNS_SUFFIX; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index d59eb166e2b..421d946c5db 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -201,8 +201,9 @@ public abstract class ControllerHttpClient { + URLEncoder.encode(name, UTF_8) + "=" + URLEncoder.encode(value, UTF_8)); } + // TODO jvenstad: remove when vaas is no longer part of region names. private static String jobNameOf(ZoneId zone) { - return zone.environment().value() + "-" + zone.region().value(); + return zone.environment().value() + "-" + zone.region().value().replaceAll("vaas-", ""); } private HttpResponse send(HttpRequest request) { -- cgit v1.2.3 From e8353801a955ba2d9a4648bf7bca019a45de944d Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Wed, 12 Jun 2019 08:18:54 +0200 Subject: Revert "Revert "Remove vaas"" --- config-provisioning/abi-spec.json | 3 +-- .../main/java/com/yahoo/config/provision/SystemName.java | 7 +------ .../java/com/yahoo/config/provision/SystemNameTest.java | 4 ++-- .../controller/api/integration/deployment/JobType.java | 4 ---- .../yahoo/vespa/hosted/controller/api/role/RoleTest.java | 16 ++++++++-------- .../vespa/hosted/controller/application/Endpoint.java | 1 - .../java/ai/vespa/hosted/api/ControllerHttpClient.java | 3 +-- 7 files changed, 13 insertions(+), 25 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/abi-spec.json b/config-provisioning/abi-spec.json index aa9b196c0e4..18f4d317019 100644 --- a/config-provisioning/abi-spec.json +++ b/config-provisioning/abi-spec.json @@ -790,8 +790,7 @@ "public static final enum com.yahoo.config.provision.SystemName main", "public static final enum com.yahoo.config.provision.SystemName Public", "public static final enum com.yahoo.config.provision.SystemName PublicCd", - "public static final enum com.yahoo.config.provision.SystemName dev", - "public static final enum com.yahoo.config.provision.SystemName vaas" + "public static final enum com.yahoo.config.provision.SystemName dev" ] }, "com.yahoo.config.provision.TenantName": { diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index ba462b9eb64..45962fc596a 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -27,10 +27,7 @@ public enum SystemName { PublicCd(true, true), /** Local development system */ - dev(false, false), - - /** VaaS */ - vaas(true, true); // TODO: Remove this and use public everywhere + dev(false, false); // TODO: Remove this and use public everywhere private final boolean isPublic; private final boolean isCd; @@ -51,7 +48,6 @@ public enum SystemName { case "main": return main; case "public": return Public; case "publiccd": return PublicCd; - case "vaas": return vaas; default: throw new IllegalArgumentException(String.format("'%s' is not a valid system", value)); } } @@ -63,7 +59,6 @@ public enum SystemName { case main: return "main"; case Public: return "public"; case PublicCd: return "publiccd"; - case vaas: return "vaas"; default : throw new IllegalStateException(); } } diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java index eb066958254..f50005edd52 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/SystemNameTest.java @@ -20,7 +20,7 @@ public class SystemNameTest { @Test public void allOf() { - assertEquals(Set.of(SystemName.cd, SystemName.PublicCd, SystemName.vaas), SystemName.allOf(SystemName::isCd)); - assertEquals(Set.of(SystemName.PublicCd, SystemName.Public, SystemName.vaas), SystemName.allOf(SystemName::isPublic)); + assertEquals(Set.of(SystemName.cd, SystemName.PublicCd), SystemName.allOf(SystemName::isCd)); + assertEquals(Set.of(SystemName.PublicCd, SystemName.Public), SystemName.allOf(SystemName::isPublic)); } } \ No newline at end of file diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 585a8f84fb2..94e111455ac 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -15,7 +15,6 @@ import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.PublicCd; import static com.yahoo.config.provision.SystemName.cd; import static com.yahoo.config.provision.SystemName.main; -import static com.yahoo.config.provision.SystemName.vaas; /** Job types that exist in the build system */ public enum JobType { @@ -88,9 +87,6 @@ public enum JobType { devCdUsCentral1 ("dev-cd-us-central-1", Map.of(cd , ZoneId.from("dev" , "cd-us-central-1"))), - devAwsUsEast1b ("dev-aws-us-east-1b", - Map.of(vaas, ZoneId.from("dev" , "vaas-aws-us-east-1b"))), - devAwsUsEast1c ("dev-aws-us-east-1c", Map.of(PublicCd, ZoneId.from("dev", "aws-us-east-1c"))), diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java index 4c11da3b697..d141ef6c73e 100644 --- a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/role/RoleTest.java @@ -17,7 +17,7 @@ import static org.junit.Assert.assertTrue; public class RoleTest { private static final Enforcer mainEnforcer = new Enforcer(SystemName.main); - private static final Enforcer vaasEnforcer = new Enforcer(SystemName.vaas); + private static final Enforcer publicEnforcer = new Enforcer(SystemName.Public); @Test public void operator_membership() { @@ -40,18 +40,18 @@ public class RoleTest { assertTrue(mainEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); Role publicSystem = Role.athenzTenantAdmin(TenantName.from("t1")); - assertFalse(vaasEnforcer.allows(publicSystem, Action.read, URI.create("/controller/v1/foo"))); - assertTrue(vaasEnforcer.allows(publicSystem, Action.read, URI.create("/badge/v1/badge"))); - assertTrue(vaasEnforcer.allows(publicSystem, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertFalse(publicEnforcer.allows(publicSystem, Action.read, URI.create("/controller/v1/foo"))); + assertTrue(publicEnforcer.allows(publicSystem, Action.read, URI.create("/badge/v1/badge"))); + assertTrue(publicEnforcer.allows(publicSystem, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); } @Test public void build_service_membership() { Role role = Role.tenantPipeline(TenantName.from("t1"), ApplicationName.from("a1")); - assertFalse(vaasEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); - assertFalse(vaasEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); - assertTrue(vaasEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport"))); - assertFalse("No global read access", vaasEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); + assertFalse(publicEnforcer.allows(role, Action.create, URI.create("/not/explicitly/defined"))); + assertFalse(publicEnforcer.allows(role, Action.update, URI.create("/application/v4/tenant/t1/application/a1"))); + assertTrue(publicEnforcer.allows(role, Action.create, URI.create("/application/v4/tenant/t1/application/a1/jobreport"))); + assertFalse("No global read access", publicEnforcer.allows(role, Action.read, URI.create("/controller/v1/foo"))); } @Test diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 6c9b8dd0784..5026ca75a83 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -152,7 +152,6 @@ public class Endpoint { if (legacy) return YAHOO_DNS_SUFFIX; return OATH_DNS_SUFFIX; case Public: - case vaas: return PUBLIC_DNS_SUFFIX; case PublicCd: return PUBLIC_CD_DNS_SUFFIX; diff --git a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java index 421d946c5db..d59eb166e2b 100644 --- a/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java +++ b/hosted-api/src/main/java/ai/vespa/hosted/api/ControllerHttpClient.java @@ -201,9 +201,8 @@ public abstract class ControllerHttpClient { + URLEncoder.encode(name, UTF_8) + "=" + URLEncoder.encode(value, UTF_8)); } - // TODO jvenstad: remove when vaas is no longer part of region names. private static String jobNameOf(ZoneId zone) { - return zone.environment().value() + "-" + zone.region().value().replaceAll("vaas-", ""); + return zone.environment().value() + "-" + zone.region().value(); } private HttpResponse send(HttpRequest request) { -- cgit v1.2.3 From 0f8d1bba78870367136257978e495fd25ab2fa19 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 12 Jun 2019 09:25:35 +0200 Subject: Remove unused curator db methods --- .../hosted/controller/persistence/CuratorDb.java | 43 ---------------------- 1 file changed, 43 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index a2e5a0c78f7..ffc410ddd0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -168,16 +168,6 @@ public class CuratorDb { return lock(lockPath(provisionStateId), Duration.ofSeconds(1)); } - @SuppressWarnings("unused") // Called by internal code - public Lock lockVespaServerPool() { - return lock(lockRoot.append("vespaServerPoolLock"), Duration.ofSeconds(1)); - } - - @SuppressWarnings("unused") // Called by internal code - public Lock lockOpenStackServerPool() { - return lock(lockRoot.append("openStackServerPoolLock"), Duration.ofSeconds(1)); - } - public Lock lockOsVersions() { return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout); } @@ -464,31 +454,6 @@ public class CuratorDb { curator.set(provisionStatePath(provisionId), data); } - @SuppressWarnings("unused") - public List readProvisionStateIds() { - return curator.getChildren(provisionStatePath()); - } - - @SuppressWarnings("unused") - public Optional readVespaServerPool() { - return curator.getData(vespaServerPoolPath()); - } - - @SuppressWarnings("unused") - public void writeVespaServerPool(byte[] data) { - curator.set(vespaServerPoolPath(), data); - } - - @SuppressWarnings("unused") - public Optional readOpenStackServerPool() { - return curator.getData(openStackServerPoolPath()); - } - - @SuppressWarnings("unused") - public void writeOpenStackServerPool(byte[] data) { - curator.set(openStackServerPoolPath(), data); - } - // -------------- Routing policies ---------------------------------------- public void writeRoutingPolicies(ApplicationId application, Set policies) { @@ -589,14 +554,6 @@ public class CuratorDb { return provisionStatePath().append(provisionId); } - private static Path vespaServerPoolPath() { - return root.append("vespaServerPool"); - } - - private static Path openStackServerPoolPath() { - return root.append("openStackServerPool"); - } - private static Path tenantPath(TenantName name) { return tenantRoot.append(name.value()); } -- cgit v1.2.3 From d64ae9a089f0f3458ed28d1800e251ec1fca5f49 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 12 Jun 2019 09:26:01 +0200 Subject: Temporarily delete the unused paths --- .../java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index ffc410ddd0d..c3aaca34fb5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -106,6 +106,10 @@ public class CuratorDb { CuratorDb(Curator curator, Duration tryLockTimeout) { this.curator = curator; this.tryLockTimeout = tryLockTimeout; + + // TODO: Remove after 7.60 + curator.delete(root.append("openStackServerPool")); + curator.delete(root.append("vespaServerPool")); } /** Returns all hosts configured to be part of this ZooKeeper cluster */ -- cgit v1.2.3 From f5a7023f55b354e8d3985cf36ea331eb765180e1 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Tue, 11 Jun 2019 16:08:21 +0200 Subject: Expose routing endpoints per cluster in RoutingGenerator --- .../controller/api/integration/routing/RoutingGenerator.java | 7 +++++++ .../hosted/controller/integration/RoutingGeneratorMock.java | 11 +++++++++++ 2 files changed, 18 insertions(+) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java index 276e19da8f6..f5c82018ac6 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/routing/RoutingGenerator.java @@ -1,9 +1,12 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.routing; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import java.net.URI; import java.util.List; +import java.util.Map; /** * @author bratseth @@ -16,4 +19,8 @@ public interface RoutingGenerator { * @return List of endpoints for that deploymentId */ List endpoints(DeploymentId deploymentId); + + /** Returns the endpoints of each cluster in the given deployment — nothing global. */ + Map clusterEndpoints(DeploymentId deploymentId); + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java index 410d7950e97..98b2dd2f7f3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java @@ -1,14 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; +import java.net.URI; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; /** * Returns a default set of endpoints on every query if it has no mappings, or those added by the user, otherwise. @@ -34,6 +37,14 @@ public class RoutingGeneratorMock implements RoutingGenerator { : routingTable.getOrDefault(deployment, Collections.emptyList()); } + @Override + public Map clusterEndpoints(DeploymentId deployment) { + return endpoints(deployment).stream() + .limit(1) + .collect(Collectors.toMap(__ -> ClusterSpec.Id.from("default"), + endpoint -> URI.create(endpoint.endpoint()))); + } + public void putEndpoints(DeploymentId deployment, List endpoints) { routingTable.put(deployment, endpoints); } -- cgit v1.2.3 From 67e75197e2e5352b38c141044cef69c4a23fc49b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 09:13:37 +0200 Subject: Expose endpoints per cluster in ApplicationController and test it for policies --- .../hosted/controller/ApplicationController.java | 26 +++++++++++++++++++++- .../maintenance/RoutingPoliciesTest.java | 16 +++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e0a73d994d1..7cdd8f3c735 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -520,7 +521,7 @@ public class ApplicationController { controller.nameServiceForwarder().createCname(RecordName.from(name), RecordData.fqdn(targetName), Priority.normal); } - /** Returns the endpoints of the deployment, or an empty list if the request fails */ + /** Returns the endpoints of the deployment, or empty if the request fails */ public Optional> getDeploymentEndpoints(DeploymentId deploymentId) { if ( ! get(deploymentId.applicationId()) .map(application -> application.deployments().containsKey(deploymentId.zoneId())) @@ -540,6 +541,29 @@ public class ApplicationController { } } + /** Returns the endpoints of the clusters in the deployment, or empty if the request fails. */ + public Optional> clusterEndpoints(DeploymentId id) { + if ( ! get(id.applicationId()) + .map(application -> application.deployments().containsKey(id.zoneId())) + .orElse(id.applicationId().instance().isTester())) + throw new NotExistsException("Deployment", id.toString()); + + try { + return Optional.of(routingPolicies.get(id)) + .filter(policies -> ! policies.isEmpty()) + .map(policies -> policies.stream() + .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) + .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + policy -> policy.endpointIn(controller.system()).url()))) + .or(() -> Optional.of(routingGenerator.clusterEndpoints(id))); + } + catch (RuntimeException e) { + log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " + + Exceptions.toMessageString(e)); + return Optional.empty(); + } + } + /** * Deletes the the given application. All known instances of the applications will be deleted, * including PR instances. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java index d94c40e6b99..be86f75e5db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java @@ -9,7 +9,9 @@ import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; @@ -18,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import org.junit.Test; +import java.net.URI; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -195,6 +198,19 @@ public class RoutingPoliciesTest { assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies().get(app1.id()).size()); } + @Test + public void cluster_endpoints_resolve_from_policies() { + provisionLoadBalancers(3, app1.id(), zone1); + tester.deployCompletely(app1, applicationPackage); + assertEquals(Map.of(ClusterSpec.Id.from("c0"), + URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), + ClusterSpec.Id.from("c1"), + URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), + ClusterSpec.Id.from("c2"), + URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), + tester.controller().applications().clusterEndpoints(new DeploymentId(app1.id(), zone1)).get()); + } + private Set policies(Application application) { return tester.controller().curator().readRoutingPolicies(application.id()); } -- cgit v1.2.3 From 5947043e5f011f4634e1f824837a19a5f1d39c74 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 11:06:09 +0200 Subject: Use cluster endpoints in test config, and test serialisation --- .../controller/deployment/InternalStepRunner.java | 51 ++++++++++++---------- .../deployment/InternalStepRunnerTest.java | 19 ++++++++ .../src/test/resources/testConfig.json | 20 +++++++++ 3 files changed, 66 insertions(+), 24 deletions(-) create mode 100644 controller-server/src/test/resources/testConfig.json (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 891a696be9c..04ff0fd2e86 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -11,8 +11,10 @@ import com.yahoo.config.application.api.Notifications.When; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.data.access.simple.JsonRender; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; @@ -28,7 +30,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; @@ -48,6 +49,7 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -320,19 +322,23 @@ public class InternalStepRunner implements StepRunner { private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) { logger.log("Attempting to find deployment endpoints ..."); - Map> endpoints = deploymentEndpoints(id, Set.of(zoneId)); + var endpoints = deploymentEndpoints(id, Set.of(zoneId)); if ( ! endpoints.containsKey(zoneId)) { logger.log("Endpoints not yet ready."); return false; } + logEndpoints(endpoints, logger); + return true; + } + + private void logEndpoints(Map> endpoints, DualLogger logger) { List messages = new ArrayList<>(); messages.add("Found endpoints:"); endpoints.forEach((zone, uris) -> { messages.add("- " + zone); - uris.forEach(uri -> messages.add(" |-- " + uri)); + uris.forEach((cluster, uri) -> messages.add(" |-- " + uri + " (" + cluster + ")")); }); logger.log(messages); - return true; } private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) { @@ -387,18 +393,12 @@ public class InternalStepRunner implements StepRunner { Set zones = testedZoneAndProductionZones(id); logger.log("Attempting to find endpoints ..."); - Map> endpoints = deploymentEndpoints(id.application(), zones); + var endpoints = deploymentEndpoints(id.application(), zones); if ( ! endpoints.containsKey(id.type().zone(controller.system())) && timedOut(deployment.get(), endpointTimeout)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); } - List messages = new ArrayList<>(); - messages.add("Found endpoints:"); - endpoints.forEach((zone, uris) -> { - messages.add("- " + zone); - uris.forEach(uri -> messages.add(" |-- " + uri)); - }); - logger.log(messages); + logEndpoints(endpoints, logger); Optional testerEndpoint = controller.jobController().testerEndpoint(id); if (testerEndpoint.isEmpty() && timedOut(deployment.get(), endpointTimeout)) { @@ -620,17 +620,12 @@ public class InternalStepRunner implements StepRunner { } /** Returns all endpoints for all current deployments of the given real application. */ - private Map> deploymentEndpoints(ApplicationId id, Iterable zones) { - ImmutableMap.Builder> deployments = ImmutableMap.builder(); - for (ZoneId zone : zones) { - controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone)) + private Map> deploymentEndpoints(ApplicationId id, Collection zones) { + ImmutableMap.Builder> deployments = ImmutableMap.builder(); + for (ZoneId zone : zones) + controller.applications().clusterEndpoints(new DeploymentId(id, zone)) .filter(endpoints -> ! endpoints.isEmpty()) - .or(() -> Optional.of(controller.applications().routingPolicies().get(id, zone).stream() - .map(policy -> policy.endpointIn(controller.system()).url()) - .collect(Collectors.toUnmodifiableList())) - .filter(endpoints -> ! endpoints.isEmpty())) .ifPresent(endpoints -> deployments.put(zone, endpoints)); - } return deployments.build(); } @@ -713,8 +708,8 @@ public class InternalStepRunner implements StepRunner { } /** Returns the config for the tests to run for the given job. */ - private static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system, - Map> deployments, Map> clusters) { + static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system, + Map> deployments, Map> clusters) { Slime slime = new Slime(); Cursor root = slime.setObject(); @@ -725,10 +720,18 @@ public class InternalStepRunner implements StepRunner { Cursor endpointsObject = root.setObject("endpoints"); deployments.forEach((zone, endpoints) -> { Cursor endpointArray = endpointsObject.setArray(zone.value()); - for (URI endpoint : endpoints) + for (URI endpoint : endpoints.values()) endpointArray.addString(endpoint.toString()); }); + Cursor zoneEndpointsObject = root.setObject("zoneEndpoints"); + deployments.forEach((zone, endpoints) -> { + Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value()); + endpoints.forEach((cluster, endpoint) -> { + clusterEndpointsObject.setString(cluster.value(), endpoint.toString()); + }); + }); + Cursor clustersObject = root.setObject("clusters"); clusters.forEach((zone, clusterList) -> { Cursor clusterArray = clustersObject.setArray(zone.value()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 370fd03d9e7..e1b4ccc0e1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Inspector; +import com.yahoo.slime.JsonFormat; +import com.yahoo.slime.ObjectTraverser; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; @@ -25,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import org.junit.Before; import org.junit.Test; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; import java.net.URI; @@ -34,6 +38,7 @@ import java.nio.file.Paths; import java.time.Duration; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.Executors; @@ -397,6 +402,20 @@ public class InternalStepRunnerTest { "1554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + "1554970337.947844\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; + @Test + public void testConfig() throws IOException { + ZoneId zone = ZoneId.from("test", "eu-north-1"); + byte[] json = InternalStepRunner.testConfig(appId, + zone, + SystemName.Public, + Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), + URI.create("https://server/"))), + Map.of(zone, List.of("facts"))); + byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes(); + assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), + new String(json)); + } + @Test public void generates_correct_services_xml_test() { assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(SystemName.cd, Optional.of("d-2-12-75")))); diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json new file mode 100644 index 00000000000..4145d863995 --- /dev/null +++ b/controller-server/src/test/resources/testConfig.json @@ -0,0 +1,20 @@ +{ + "application": "tenant:application:default", + "zone": "test.aws-us-east-1c", + "system": "publiccd", + "endpoints": { + "test.aws-us-east-1c": [ + "https://server/" + ] + }, + "zoneEndpoints": { + "test.aws-us-east-1c": { + "ai": "https://server/" + } + }, + "clusters": { + "test.aws-us-east-1c": [ + "facts" + ] + } +} -- cgit v1.2.3 From 64755b214c236b41664e47c8a38dc98982cff52b Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 11:56:05 +0200 Subject: Move things around --- .../hosted/controller/ApplicationController.java | 12 +++- .../controller/deployment/InternalStepRunner.java | 74 +++------------------- .../controller/deployment/JobController.java | 9 +++ .../deployment/TestConfigSerializer.java | 73 +++++++++++++++++++++ .../deployment/InternalStepRunnerTest.java | 14 ---- .../deployment/TestConfigSerializerTest.java | 36 +++++++++++ 6 files changed, 138 insertions(+), 80 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7cdd8f3c735..6eadfbd5974 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -77,6 +77,7 @@ import java.security.Principal; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; @@ -88,6 +89,7 @@ import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.function.Consumer; +import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -541,7 +543,7 @@ public class ApplicationController { } } - /** Returns the endpoints of the clusters in the deployment, or empty if the request fails. */ + /** Returns the non-empty endpoints of the clusters in the deployment, or empty if the request fails. */ public Optional> clusterEndpoints(DeploymentId id) { if ( ! get(id.applicationId()) .map(application -> application.deployments().containsKey(id.zoneId())) @@ -564,6 +566,14 @@ public class ApplicationController { } } + /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ + public Map> clusterEndpoints(ApplicationId id, Collection zones) { + Map> deployments = new HashMap<>(); + for (ZoneId zone : zones) + clusterEndpoints(new DeploymentId(id, zone)).ifPresent(endpoints -> deployments.put(zone, endpoints)); + return Map.copyOf(deployments); + } + /** * Deletes the the given application. All known instances of the applications will be deleted, * including PR instances. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 04ff0fd2e86..82402205355 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -14,7 +14,6 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.data.access.simple.JsonRender; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; import com.yahoo.slime.Cursor; @@ -49,7 +48,6 @@ import java.net.URI; import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -101,10 +99,12 @@ public class InternalStepRunner implements StepRunner { static final Duration installationTimeout = Duration.ofMinutes(150); private final Controller controller; + private final TestConfigSerializer testConfigSerializer; private final DeploymentFailureMails mails; public InternalStepRunner(Controller controller) { this.controller = controller; + this.testConfigSerializer = new TestConfigSerializer(controller.system()); this.mails = new DeploymentFailureMails(controller.zoneRegistry()); } @@ -322,7 +322,7 @@ public class InternalStepRunner implements StepRunner { private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) { logger.log("Attempting to find deployment endpoints ..."); - var endpoints = deploymentEndpoints(id, Set.of(zoneId)); + var endpoints = controller.applications().clusterEndpoints(id, Set.of(zoneId)); if ( ! endpoints.containsKey(zoneId)) { logger.log("Endpoints not yet ready."); return false; @@ -390,10 +390,10 @@ public class InternalStepRunner implements StepRunner { return Optional.of(aborted); } - Set zones = testedZoneAndProductionZones(id); + Set zones = controller.jobController().testedZoneAndProductionZones(id.application(), id.type()); logger.log("Attempting to find endpoints ..."); - var endpoints = deploymentEndpoints(id.application(), zones); + var endpoints = controller.applications().clusterEndpoints(id.application(), zones); if ( ! endpoints.containsKey(id.type().zone(controller.system())) && timedOut(deployment.get(), endpointTimeout)) { logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!"); return Optional.of(error); @@ -410,9 +410,10 @@ public class InternalStepRunner implements StepRunner { logger.log("Starting tests ..."); controller.jobController().cloud().startTests(testerEndpoint.get(), TesterCloud.Suite.of(id.type()), - testConfig(id.application(), id.type().zone(controller.system()), - controller.system(), endpoints, - listClusters(id.application(), zones))); + testConfigSerializer.testConfig(id.application(), + id.type(), + endpoints, + listClusters(id.application(), zones))); return Optional.of(running); } @@ -612,23 +613,6 @@ public class InternalStepRunner implements StepRunner { throw new IllegalStateException("No step deploys to the zone this run is for!"); } - /** Returns a stream containing the zone of the deployment tested in the given run, and all production zones for the application. */ - private Set testedZoneAndProductionZones(RunId id) { - return Stream.concat(Stream.of(id.type().zone(controller.system())), - application(id.application()).productionDeployments().keySet().stream()) - .collect(Collectors.toSet()); - } - - /** Returns all endpoints for all current deployments of the given real application. */ - private Map> deploymentEndpoints(ApplicationId id, Collection zones) { - ImmutableMap.Builder> deployments = ImmutableMap.builder(); - for (ZoneId zone : zones) - controller.applications().clusterEndpoints(new DeploymentId(id, zone)) - .filter(endpoints -> ! endpoints.isEmpty()) - .ifPresent(endpoints -> deployments.put(zone, endpoints)); - return deployments.build(); - } - /** Returns all content clusters in all current deployments of the given real application. */ private Map> listClusters(ApplicationId id, Iterable zones) { ImmutableMap.Builder> clusters = ImmutableMap.builder(); @@ -707,46 +691,6 @@ public class InternalStepRunner implements StepRunner { return deploymentSpec.getBytes(StandardCharsets.UTF_8); } - /** Returns the config for the tests to run for the given job. */ - static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system, - Map> deployments, Map> clusters) { - Slime slime = new Slime(); - Cursor root = slime.setObject(); - - root.setString("application", id.serializedForm()); - root.setString("zone", testerZone.value()); - root.setString("system", system.value()); - - Cursor endpointsObject = root.setObject("endpoints"); - deployments.forEach((zone, endpoints) -> { - Cursor endpointArray = endpointsObject.setArray(zone.value()); - for (URI endpoint : endpoints.values()) - endpointArray.addString(endpoint.toString()); - }); - - Cursor zoneEndpointsObject = root.setObject("zoneEndpoints"); - deployments.forEach((zone, endpoints) -> { - Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value()); - endpoints.forEach((cluster, endpoint) -> { - clusterEndpointsObject.setString(cluster.value(), endpoint.toString()); - }); - }); - - Cursor clustersObject = root.setObject("clusters"); - clusters.forEach((zone, clusterList) -> { - Cursor clusterArray = clustersObject.setArray(zone.value()); - for (String cluster : clusterList) - clusterArray.addString(cluster); - }); - - try { - return SlimeUtils.toJsonBytes(slime); - } - catch (IOException e) { - throw new UncheckedIOException(e); - } - } - /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */ private class DualLogger { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 5bb9686ce0f..e9bcb714d44 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -41,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.function.UnaryOperator; import java.util.logging.Level; +import java.util.stream.Collectors; import java.util.stream.Stream; import static com.google.common.collect.ImmutableList.copyOf; @@ -414,6 +416,13 @@ public class JobController { .map(policy -> policy.endpointIn(controller.system()).url())); } + /** Returns a set containing the zone of the deployment tested in the given run, and all production zones for the application. */ + public Set testedZoneAndProductionZones(ApplicationId id, JobType type) { + return Stream.concat(Stream.of(type.zone(controller.system())), + controller.applications().require(id).productionDeployments().keySet().stream()) + .collect(Collectors.toSet()); + } + // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service. private long nextBuild(ApplicationId id) { return 1 + controller.applications().require(id).deploymentJobs() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java new file mode 100644 index 00000000000..24ac7268f33 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -0,0 +1,73 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +/** + * Serializes config for integration tests against Vespa deployments. + * + * @author jonmv + */ +public class TestConfigSerializer { + + private final SystemName system; + + public TestConfigSerializer(SystemName system) { + this.system = system; + } + + /** Returns the config for the tests to run for the given job. */ + public byte[] testConfig(ApplicationId id, + JobType type, + Map> deployments, + Map> clusters) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + root.setString("application", id.serializedForm()); + root.setString("zone", type.zone(system).value()); + root.setString("system", system.value()); + + Cursor endpointsObject = root.setObject("endpoints"); // TODO jvenstad: remove. + deployments.forEach((zone, endpoints) -> { + Cursor endpointArray = endpointsObject.setArray(zone.value()); + for (URI endpoint : endpoints.values()) + endpointArray.addString(endpoint.toString()); + }); + + Cursor zoneEndpointsObject = root.setObject("zoneEndpoints"); + deployments.forEach((zone, endpoints) -> { + Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value()); + endpoints.forEach((cluster, endpoint) -> { + clusterEndpointsObject.setString(cluster.value(), endpoint.toString()); + }); + }); + + Cursor clustersObject = root.setObject("clusters"); + clusters.forEach((zone, clusterList) -> { + Cursor clusterArray = clustersObject.setArray(zone.value()); + for (String cluster : clusterList) + clusterArray.addString(cluster); + }); + + try { + return SlimeUtils.toJsonBytes(slime); + } + catch (IOException e) { + throw new UncheckedIOException(e); + } + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index e1b4ccc0e1f..4fb6a2e34fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -402,20 +402,6 @@ public class InternalStepRunnerTest { "1554970337.947820\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstdout\tinfo\tERROR: Bundle canary-application [71] Unable to get module class path. (java.lang.NullPointerException)\n" + "1554970337.947844\t17491290-v6-1.ostk.bm2.prod.ne1.yahoo.com\t5480\tcontainer\tstderr\twarning\tjava.lang.NullPointerException\\n\\tat org.apache.felix.framework.BundleRevisionImpl.calculateContentPath(BundleRevisionImpl.java:438)\\n\\tat org.apache.felix.framework.BundleRevisionImpl.initializeContentPath(BundleRevisionImpl.java:371)"; - @Test - public void testConfig() throws IOException { - ZoneId zone = ZoneId.from("test", "eu-north-1"); - byte[] json = InternalStepRunner.testConfig(appId, - zone, - SystemName.Public, - Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), - URI.create("https://server/"))), - Map.of(zone, List.of("facts"))); - byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes(); - assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), - new String(json)); - } - @Test public void generates_correct_services_xml_test() { assertFile("test_runner_services.xml-cd", new String(InternalStepRunner.servicesXml(SystemName.cd, Optional.of("d-2-12-75")))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java new file mode 100644 index 00000000000..877cfa3b59b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -0,0 +1,36 @@ +package com.yahoo.vespa.hosted.controller.deployment; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import org.junit.Test; + +import java.io.IOException; +import java.net.URI; +import java.util.List; +import java.util.Map; + +import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId; +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class TestConfigSerializerTest { + + @Test + public void testConfig() throws IOException { + ZoneId zone = JobType.systemTest.zone(SystemName.PublicCd); + byte[] json = new TestConfigSerializer(SystemName.PublicCd).testConfig(appId, + JobType.systemTest, + Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), + URI.create("https://server/"))), + Map.of(zone, List.of("facts"))); + byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes(); + assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), + new String(json)); + } + +} -- cgit v1.2.3 From 437f5d3d7f9dea815e34ccc0d00694ed5bb8cea9 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 13:04:16 +0200 Subject: Filter on emptiness of endpoint map --- .../com/yahoo/vespa/hosted/controller/ApplicationController.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 6eadfbd5974..c42363033aa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -543,7 +543,7 @@ public class ApplicationController { } } - /** Returns the non-empty endpoints of the clusters in the deployment, or empty if the request fails. */ + /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */ public Optional> clusterEndpoints(DeploymentId id) { if ( ! get(id.applicationId()) .map(application -> application.deployments().containsKey(id.zoneId())) @@ -557,7 +557,8 @@ public class ApplicationController { .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), policy -> policy.endpointIn(controller.system()).url()))) - .or(() -> Optional.of(routingGenerator.clusterEndpoints(id))); + .or(() -> Optional.of(routingGenerator.clusterEndpoints(id))) + .filter(endpoints -> ! endpoints.isEmpty()); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " -- cgit v1.2.3 From 5179be6d7834daa9e4b054a27f31858be9e864d7 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 13:36:09 +0200 Subject: Expose test config under application/v4 --- .../controller/deployment/InternalStepRunner.java | 7 +-- .../deployment/TestConfigSerializer.java | 27 +++++--- .../restapi/application/ApplicationApiHandler.java | 11 ++++ .../deployment/TestConfigSerializerTest.java | 2 +- .../restapi/application/ApplicationApiTest.java | 11 +++- .../restapi/application/responses/application.json | 4 +- .../responses/application1-recursive.json | 2 +- .../responses/delete-with-active-deployments.json | 2 +- .../application/responses/dev-us-east-1.json | 71 ++++++++++++++++++++++ .../application/responses/dev-us-west-1.json | 71 ---------------------- .../restapi/application/responses/test-config.json | 21 +++++++ 11 files changed, 135 insertions(+), 94 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 82402205355..bf2460284ab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -16,9 +16,6 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.io.IOUtils; import com.yahoo.log.LogLevel; -import com.yahoo.slime.Cursor; -import com.yahoo.slime.Slime; -import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.ActivateResult; @@ -41,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport; import com.yahoo.yolean.Exceptions; import java.io.ByteArrayOutputStream; -import java.io.IOException; import java.io.PrintStream; import java.io.UncheckedIOException; import java.net.URI; @@ -57,7 +53,6 @@ import java.util.function.Supplier; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; @@ -410,7 +405,7 @@ public class InternalStepRunner implements StepRunner { logger.log("Starting tests ..."); controller.jobController().cloud().startTests(testerEndpoint.get(), TesterCloud.Suite.of(id.type()), - testConfigSerializer.testConfig(id.application(), + testConfigSerializer.configJson(id.application(), id.type(), endpoints, listClusters(id.application(), zones))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java index 24ac7268f33..e79692d34ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java @@ -28,8 +28,7 @@ public class TestConfigSerializer { this.system = system; } - /** Returns the config for the tests to run for the given job. */ - public byte[] testConfig(ApplicationId id, + public Slime configSlime(ApplicationId id, JobType type, Map> deployments, Map> clusters) { @@ -55,15 +54,25 @@ public class TestConfigSerializer { }); }); - Cursor clustersObject = root.setObject("clusters"); - clusters.forEach((zone, clusterList) -> { - Cursor clusterArray = clustersObject.setArray(zone.value()); - for (String cluster : clusterList) - clusterArray.addString(cluster); - }); + if ( ! clusters.isEmpty()) { + Cursor clustersObject = root.setObject("clusters"); + clusters.forEach((zone, clusterList) -> { + Cursor clusterArray = clustersObject.setArray(zone.value()); + for (String cluster : clusterList) + clusterArray.addString(cluster); + }); + } + + return slime; + } + /** Returns the config for the tests to run for the given job. */ + public byte[] configJson(ApplicationId id, + JobType type, + Map> deployments, + Map> clusters) { try { - return SlimeUtils.toJsonBytes(slime); + return SlimeUtils.toJsonBytes(configSlime(id, type, deployments, clusters)); } catch (IOException e) { throw new UncheckedIOException(e); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 78148bf5428..9a080d9c63c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -66,6 +66,7 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel; +import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; @@ -95,6 +96,7 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Base64; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -182,6 +184,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -1109,6 +1112,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } } + private HttpResponse testConfig(ApplicationId id, JobType type) { + var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type)); + return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id, + type, + endpoints, + Collections.emptyMap())); + } + private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) { Optional jobError = Optional.empty(); if (report.field("jobError").valid()) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index 877cfa3b59b..bc411d4377d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -23,7 +23,7 @@ public class TestConfigSerializerTest { @Test public void testConfig() throws IOException { ZoneId zone = JobType.systemTest.zone(SystemName.PublicCd); - byte[] json = new TestConfigSerializer(SystemName.PublicCd).testConfig(appId, + byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(appId, JobType.systemTest, Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), URI.create("https://server/"))), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 99bc07a9528..632fea8cb1f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -209,7 +209,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification) MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default/deploy", POST) .data(entity) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data))) .userIdentity(USER_ID), @@ -521,10 +521,15 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT), new File("delete-with-active-deployments.json"), 400); + // GET config for running a test against a deployment + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/dev-us-east-1/test-config", GET) + .userIdentity(USER_ID), + new File("test-config.json")); + // DELETE (deactivate) a deployment - dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default", DELETE) .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default"); // DELETE (deactivate) a deployment - prod tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", DELETE) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 4b2cb397b5b..ce29165d497 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -225,9 +225,9 @@ "instances": [ { "environment": "dev", - "region": "us-west-1", + "region": "us-east-1", "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default" }, { "bcpStatus": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index fa903b61825..de99019447a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -223,7 +223,7 @@ ], "rotationId": "rotation-id-1", "instances": [ - @include(dev-us-west-1.json), + @include(dev-us-east-1.json), @include(prod-us-central-1.json) ], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json index 2b17d55627a..012b0031489 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-west-1 in default, zone prod.us-central-1 in default" + "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-east-1 in default, zone prod.us-central-1 in default" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json new file mode 100644 index 00000000000..7cbbe4be43b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json @@ -0,0 +1,71 @@ +{ + "tenant": "tenant1", + "application": "application1", + "instance": "default", + "environment": "dev", + "region": "us-east-1", + "endpoints": [], + "serviceUrls": [ + "http://old-endpoint.vespa.yahooapis.com:4080", + "http://qrs-endpoint.vespa.yahooapis.com:4080", + "http://feeding-endpoint.vespa.yahooapis.com:4080", + "http://global-endpoint.vespa.yahooapis.com:4080", + "http://alias-endpoint.vespa.yahooapis.com:4080" + ], + "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-east-1&application=tenant1.application1", + "version": "(ignore)", + "revision": "(ignore)", + "deployTimeEpochMs": "(ignore)", + "screwdriverId": "123", + "activity": { + "lastQueried": 1527848130000, + "lastWritten": 1527848130000, + "lastQueriesPerSecond": 1.0, + "lastWritesPerSecond": 2.0 + }, + "cost": { + "tco": 74, + "waste": 0, + "utilization": 2.9999999999999996, + "cluster": { + "cluster1": { + "count": 2, + "resource": "cpu", + "utilization": 2.9999999999999996, + "tco": 74, + "waste": 0, + "flavor": "flavor1", + "flavorCost": 37.0, + "flavorCpu": 2.0, + "flavorMem": 4.0, + "flavorDisk": 50.0, + "type": "content", + "util": { + "cpu": 2.9999999999999996, + "mem": 0.4285714285714286, + "disk": 0.5714285714285715, + "diskBusy": 1.0 + }, + "usage": { + "cpu": 0.6, + "mem": 0.3, + "disk": 0.4, + "diskBusy": 0.3 + }, + "hostnames": [ + "host1", + "host2" + ] + } + } + }, + "metrics": { + "queriesPerSecond": 1.0, + "writesPerSecond": 2.0, + "documentCount": 3.0, + "queryLatencyMillis": 4.0, + "writeLatencyMillis": 5.0, + "lastUpdated": 123123 + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json deleted file mode 100644 index 67c71ee3880..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json +++ /dev/null @@ -1,71 +0,0 @@ -{ - "tenant": "tenant1", - "application": "application1", - "instance": "default", - "environment": "dev", - "region": "us-west-1", - "endpoints": [], - "serviceUrls": [ - "http://old-endpoint.vespa.yahooapis.com:4080", - "http://qrs-endpoint.vespa.yahooapis.com:4080", - "http://feeding-endpoint.vespa.yahooapis.com:4080", - "http://global-endpoint.vespa.yahooapis.com:4080", - "http://alias-endpoint.vespa.yahooapis.com:4080" - ], - "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-west-1&application=tenant1.application1", - "version": "(ignore)", - "revision": "(ignore)", - "deployTimeEpochMs": "(ignore)", - "screwdriverId": "123", - "activity": { - "lastQueried": 1527848130000, - "lastWritten": 1527848130000, - "lastQueriesPerSecond": 1.0, - "lastWritesPerSecond": 2.0 - }, - "cost": { - "tco": 74, - "waste": 0, - "utilization": 2.9999999999999996, - "cluster": { - "cluster1": { - "count": 2, - "resource": "cpu", - "utilization": 2.9999999999999996, - "tco": 74, - "waste": 0, - "flavor": "flavor1", - "flavorCost": 37.0, - "flavorCpu": 2.0, - "flavorMem": 4.0, - "flavorDisk": 50.0, - "type": "content", - "util": { - "cpu": 2.9999999999999996, - "mem": 0.4285714285714286, - "disk": 0.5714285714285715, - "diskBusy": 1.0 - }, - "usage": { - "cpu": 0.6, - "mem": 0.3, - "disk": 0.4, - "diskBusy": 0.3 - }, - "hostnames": [ - "host1", - "host2" - ] - } - } - }, - "metrics": { - "queriesPerSecond": 1.0, - "writesPerSecond": 2.0, - "documentCount": 3.0, - "queryLatencyMillis": 4.0, - "writeLatencyMillis": 5.0, - "lastUpdated": 123123 - } -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json new file mode 100644 index 00000000000..fef3cf6a372 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json @@ -0,0 +1,21 @@ +{ + "application": "tenant1:application1:default", + "zone": "dev.us-east-1", + "system": "main", + "endpoints": { + "dev.us-east-1": [ + "http://old-endpoint.vespa.yahooapis.com:4080" + ], + "prod.us-central-1": [ + "http://old-endpoint.vespa.yahooapis.com:4080" + ] + }, + "zoneEndpoints": { + "dev.us-east-1": { + "default": "http://old-endpoint.vespa.yahooapis.com:4080" + }, + "prod.us-central-1": { + "default": "http://old-endpoint.vespa.yahooapis.com:4080" + } + } +} -- cgit v1.2.3 From 5a742d376f0668ad18d0bf443299d9a4a646d10d Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 13:52:01 +0200 Subject: Sort on zone to ensure consistent order in unit test --- .../com/yahoo/vespa/hosted/controller/ApplicationController.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c42363033aa..005e1a4cd01 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -88,6 +88,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.TreeMap; import java.util.function.Consumer; import java.util.function.Predicate; import java.util.logging.Level; @@ -569,10 +570,10 @@ public class ApplicationController { /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ public Map> clusterEndpoints(ApplicationId id, Collection zones) { - Map> deployments = new HashMap<>(); + Map> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value)); for (ZoneId zone : zones) clusterEndpoints(new DeploymentId(id, zone)).ifPresent(endpoints -> deployments.put(zone, endpoints)); - return Map.copyOf(deployments); + return Collections.unmodifiableMap(deployments); } /** -- cgit v1.2.3 From 065b03e8e0eadbf72c683b0ee9539dd8a679ce8b Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 12 Jun 2019 16:19:07 +0200 Subject: Add accidentally removed method --- .../com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index c3aaca34fb5..d704d701cf0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -458,6 +458,11 @@ public class CuratorDb { curator.set(provisionStatePath(provisionId), data); } + @SuppressWarnings("unused") + public List readProvisionStateIds() { + return curator.getChildren(provisionStatePath()); + } + // -------------- Routing policies ---------------------------------------- public void writeRoutingPolicies(ApplicationId application, Set policies) { -- cgit v1.2.3 From 678c4ab12601da7e63518f240e3d17c8cd8b8a46 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Wed, 12 Jun 2019 22:19:48 +0200 Subject: Avoid Optional> --- .../hosted/controller/ApplicationController.java | 26 ++++++++++++---------- .../controller/deployment/JobController.java | 2 +- .../restapi/application/ApplicationApiHandler.java | 2 +- .../maintenance/RoutingPoliciesTest.java | 2 +- 4 files changed, 17 insertions(+), 15 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 005e1a4cd01..a079c8df78e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -525,27 +525,27 @@ public class ApplicationController { } /** Returns the endpoints of the deployment, or empty if the request fails */ - public Optional> getDeploymentEndpoints(DeploymentId deploymentId) { + public List getDeploymentEndpoints(DeploymentId deploymentId) { if ( ! get(deploymentId.applicationId()) .map(application -> application.deployments().containsKey(deploymentId.zoneId())) .orElse(deploymentId.applicationId().instance().isTester())) throw new NotExistsException("Deployment", deploymentId.toString()); try { - return Optional.of(ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() - .map(RoutingEndpoint::endpoint) - .map(URI::create) - .iterator())); + return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() + .map(RoutingEndpoint::endpoint) + .map(URI::create) + .iterator()); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId + ": " + Exceptions.toMessageString(e)); - return Optional.empty(); + return Collections.emptyList(); } } /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */ - public Optional> clusterEndpoints(DeploymentId id) { + public Map clusterEndpoints(DeploymentId id) { if ( ! get(id.applicationId()) .map(application -> application.deployments().containsKey(id.zoneId())) .orElse(id.applicationId().instance().isTester())) @@ -558,21 +558,23 @@ public class ApplicationController { .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), policy -> policy.endpointIn(controller.system()).url()))) - .or(() -> Optional.of(routingGenerator.clusterEndpoints(id))) - .filter(endpoints -> ! endpoints.isEmpty()); + .orElseGet(() -> routingGenerator.clusterEndpoints(id)); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " + Exceptions.toMessageString(e)); - return Optional.empty(); + return Collections.emptyMap(); } } /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ public Map> clusterEndpoints(ApplicationId id, Collection zones) { Map> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value)); - for (ZoneId zone : zones) - clusterEndpoints(new DeploymentId(id, zone)).ifPresent(endpoints -> deployments.put(zone, endpoints)); + for (ZoneId zone : zones) { + var endpoints = clusterEndpoints(new DeploymentId(id, zone)); + if ( ! endpoints.isEmpty()) + deployments.put(zone, endpoints); + } return Collections.unmodifiableMap(deployments); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index e9bcb714d44..c644af2e554 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -410,7 +410,7 @@ public class JobController { Optional testerEndpoint(RunId id) { DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); return controller.applications().getDeploymentEndpoints(testerId) - .flatMap(uris -> uris.stream().findAny()) + .stream().findAny() .or(() -> controller.applications().routingPolicies().get(testerId).stream() .findAny() .map(policy -> policy.endpointIn(controller.system()).url())); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9a080d9c63c..4402a6267fe 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -602,7 +602,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // ask the routing layer here Cursor serviceUrlArray = response.setArray("serviceUrls"); controller.applications().getDeploymentEndpoints(deploymentId) - .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()))); + .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())); response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString()); response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java index be86f75e5db..f19c309dd67 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java @@ -208,7 +208,7 @@ public class RoutingPoliciesTest { URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"), ClusterSpec.Id.from("c2"), URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")), - tester.controller().applications().clusterEndpoints(new DeploymentId(app1.id(), zone1)).get()); + tester.controller().applications().clusterEndpoints(new DeploymentId(app1.id(), zone1))); } private Set policies(Application application) { -- cgit v1.2.3 From 85badff5f9bc6c9cc5eb441dc2a5bfda88bb4d4b Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 13 Jun 2019 10:43:40 +0200 Subject: Remove artificial distinction between cnames and rotation names --- .../api/integration/configserver/ConfigServer.java | 2 +- .../hosted/controller/ApplicationController.java | 24 ++++++++++------------ .../vespa/hosted/controller/ControllerTest.java | 2 +- .../controller/integration/ConfigServerMock.java | 11 +++++----- 4 files changed, 18 insertions(+), 21 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 03b3d586b73..0c6bee4073d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -31,7 +31,7 @@ public interface ConfigServer { PrepareResponse prepareResponse(); } - PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationCnames, Set rotationNames, byte[] content); + PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, byte[] content); void restart(DeploymentId deployment, Optional hostname); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index e0a73d994d1..72ab32280f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -208,7 +208,7 @@ public class ApplicationController { return findGlobalEndpoint(deployment).map(endpoint -> { try { EndpointStatus status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName()); - return Collections.singletonMap(endpoint, status); + return Map.of(endpoint, status); } catch (IOException e) { throw new UncheckedIOException("Failed to get rotation status of " + deployment, e); } @@ -284,7 +284,6 @@ public class ApplicationController { ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set rotationNames = new HashSet<>(); - Set cnames; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -326,9 +325,9 @@ public class ApplicationController { application = withRotation(application, zone); Application app = application.get(); // Include global DNS names - cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet()); + app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(rotationNames::add); // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header - app.rotations().stream().map(RotationId::asString).forEach(cnames::add); + app.rotations().stream().map(RotationId::asString).forEach(rotationNames::add); // Update application with information from application package if ( ! preferOldestVersion @@ -340,7 +339,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, cnames); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -407,7 +406,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of()); + return deploy(application.id(), applicationPackage, zone, options, Set.of()); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -415,16 +414,15 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet()); + return deploy(tester.id(), applicationPackage, zone, options, Set.of()); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set rotationNames, Set cnames) { + Set rotationNames) { DeploymentId deploymentId = new DeploymentId(application, zone); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, cnames, rotationNames, - applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, rotationNames, applicationPackage.zippedContent()); // Refresh routing policies on successful deployment. At this point we can safely assume that the config server // has allocated load balancers for the deployment. @@ -468,8 +466,8 @@ public class ApplicationController { logEntry.message = "Ignoring deployment of application '" + application + "' to " + zone + " as a deployment is not currently expected"; PrepareResponse prepareResponse = new PrepareResponse(); - prepareResponse.log = Collections.singletonList(logEntry); - prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList()); + prepareResponse.log = List.of(logEntry); + prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of()); return new ActivateResult(new RevisionId("0"), prepareResponse, 0); } @@ -780,7 +778,7 @@ public class ApplicationController { if (!"warn".equalsIgnoreCase(log.level) && !"warning".equalsIgnoreCase(log.level)) continue; warnings.merge(DeploymentMetrics.Warning.all, 1, Integer::sum); } - return Collections.unmodifiableMap(warnings); + return Map.copyOf(warnings); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 3fca94ef21f..de31f1f67f9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -293,7 +293,7 @@ public class ControllerTest { "app1--tenant1.global.vespa.oath.cloud", "app1.tenant1.global.vespa.yahooapis.com", "app1--tenant1.global.vespa.yahooapis.com"), - tester.configServer().rotationCnames().get(new DeploymentId(application.id(), deployment.zone()))); + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); } tester.flushDnsRequests(); assertEquals(3, tester.controllerTester().nameService().records().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 1894a51adc3..cb39e066e16 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -59,7 +59,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Set suspendedApplications = new HashSet<>(); private final Map> loadBalancers = new HashMap<>(); private final Map> warnings = new HashMap<>(); - private final Map> rotationCnames = new HashMap<>(); + private final Map> rotationNames = new HashMap<>(); private Version lastPrepareVersion = null; private RuntimeException prepareException = null; @@ -180,8 +180,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer warnings.put(deployment, List.copyOf(logs)); } - public Map> rotationCnames() { - return Collections.unmodifiableMap(rotationCnames); + public Map> rotationNames() { + return Collections.unmodifiableMap(rotationNames); } @Override @@ -223,8 +223,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationCnames, - Set rotationNames, byte[] content) { + public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; @@ -236,7 +235,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) provision(deployment.zoneId(), deployment.applicationId()); - this.rotationCnames.put(deployment, Set.copyOf(rotationCnames)); + this.rotationNames.put(deployment, Set.copyOf(rotationNames)); return new PreparedApplication() { -- cgit v1.2.3 From 3866b2dd2537168a752354f87ec3e8095a96f890 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 13 Jun 2019 12:22:39 +0200 Subject: Log name service request --- .../java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java | 1 + 1 file changed, 1 insertion(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java index 684fb091d92..4768577aa7b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java @@ -74,6 +74,7 @@ public class NameServiceQueue { var queue = new NameServiceQueue(requests); for (int i = 0; i < n && !queue.requests.isEmpty(); i++) { var request = queue.requests.peek(); + log.log(LogLevel.INFO, "Dispatching name service request: " + request); try { request.dispatchTo(nameService); queue.requests.poll(); -- cgit v1.2.3 From bd1162996389649436332ef2d4a2acb1ac562503 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 13 Jun 2019 12:47:32 +0200 Subject: application/v4 go-forward functionality for any instance --- .../restapi/application/ApplicationApiHandler.java | 103 ++++++++++++--------- .../application/JobControllerApiHandlerHelper.java | 8 +- 2 files changed, 64 insertions(+), 47 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 78148bf5428..0f513854206 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -174,12 +174,16 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/user")) return authenticatedUser(request); if (path.matches("/application/v4/tenant")) return tenants(request); if (path.matches("/application/v4/tenant/{tenant}")) return tenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application")) return applications(path.get("tenant"), Optional.empty(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return application(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); @@ -202,14 +206,19 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), false, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), true, request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return submit(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return createApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{jobtype}")) return jobDeploy(appIdFromPath(path), jobTypeFromPath(path), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -220,17 +229,21 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } private HttpResponse handlePATCH(Path path, HttpRequest request) { - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) - return patchApplication(path.get("tenant"), path.get("application"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return patchApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return patchApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } private HttpResponse handleDELETE(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return deleteTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "all"); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("choice")); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return deleteApplication(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", "all"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), "default", path.get("choice")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), "default"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}")) return deleteApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), "all"); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), path.get("instance")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) @@ -299,25 +312,28 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse applications(String tenantName, HttpRequest request) { + private HttpResponse applications(String tenantName, Optional applicationName, HttpRequest request) { TenantName tenant = TenantName.from(tenantName); Slime slime = new Slime(); Cursor array = slime.setArray(); - for (Application application : controller.applications().asList(tenant)) + for (Application application : controller.applications().asList(tenant)) { + if (applicationName.isPresent() && ! application.id().application().toString().equals(applicationName.get())) + continue; toSlime(application, array.addObject(), request); + } return new SlimeJsonResponse(slime); } - private HttpResponse application(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse application(String tenantName, String applicationName, String instanceName, HttpRequest request) { Slime slime = new Slime(); - toSlime(slime.setObject(), getApplication(tenantName, applicationName), request); + toSlime(slime.setObject(), getApplication(tenantName, applicationName, instanceName), request); return new SlimeJsonResponse(slime); } - private HttpResponse patchApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse patchApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); StringJoiner messageBuilder = new StringJoiner("\n").setEmptyValue("No applicable changes."); - controller.applications().lockOrThrow(ApplicationId.from(tenantName, applicationName, "default"), application -> { + controller.applications().lockOrThrow(ApplicationId.from(tenantName, applicationName, instanceName), application -> { Inspector majorVersionField = requestObject.field("majorVersion"); if (majorVersionField.valid()) { Integer majorVersion = majorVersionField.asLong() == 0 ? null : (int) majorVersionField.asLong(); @@ -337,8 +353,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new MessageResponse(messageBuilder.toString()); } - private Application getApplication(String tenantName, String applicationName) { - ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, "default"); + private Application getApplication(String tenantName, String applicationName, String instanceName) { + ApplicationId applicationId = ApplicationId.from(tenantName, applicationName, instanceName); return controller.applications().get(applicationId) .orElseThrow(() -> new NotExistsException(applicationId + " not found")); } @@ -722,11 +738,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - private HttpResponse deploying(String tenant, String application, HttpRequest request) { - Application app = controller.applications().require(ApplicationId.from(tenant, application, "default")); + private HttpResponse deploying(String tenant, String application, String instance, HttpRequest request) { + Application app = controller.applications().require(ApplicationId.from(tenant, application, instance)); Slime slime = new Slime(); Cursor root = slime.setObject(); - if (!app.change().isEmpty()) { + if ( ! app.change().isEmpty()) { app.change().platform().ifPresent(version -> root.setString("platform", version.toString())); app.change().application().ifPresent(applicationVersion -> root.setString("application", applicationVersion.id())); root.setBool("pinned", app.change().isPinned()); @@ -801,9 +817,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(controller.tenants().require(TenantName.from(tenantName)), request); } - private HttpResponse createApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse createApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { Inspector requestObject = toSlime(request.getData()).get(); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); try { Optional credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() @@ -823,10 +839,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Trigger deployment of the given Vespa version if a valid one is given, e.g., "7.8.9". */ - private HttpResponse deployPlatform(String tenantName, String applicationName, boolean pin, HttpRequest request) { + private HttpResponse deployPlatform(String tenantName, String applicationName, String instanceName, boolean pin, HttpRequest request) { request = controller.auditLogger().log(request); String versionString = readToString(request.getData()); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Version version = Version.fromString(versionString); @@ -834,12 +850,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler { version = controller.systemVersion(); if ( ! systemHasVersion(version)) throw new IllegalArgumentException("Cannot trigger deployment of version '" + version + "': " + - "Version is not active in this system. " + - "Active versions: " + controller.versionStatus().versions() - .stream() - .map(VespaVersion::versionNumber) - .map(Version::toString) - .collect(joining(", "))); + "Version is not active in this system. " + + "Active versions: " + controller.versionStatus().versions() + .stream() + .map(VespaVersion::versionNumber) + .map(Version::toString) + .collect(joining(", "))); Change change = Change.of(version); if (pin) change = change.withPin(); @@ -851,9 +867,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Trigger deployment to the last known application package for the given application. */ - private HttpResponse deployApplication(String tenantName, String applicationName, HttpRequest request) { + private HttpResponse deployApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { controller.auditLogger().log(request); - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = Change.of(application.get().deploymentJobs().statusOf(JobType.component).get().lastSuccess().get().application()); @@ -864,8 +880,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** Cancel ongoing change for given application, e.g., everything with {"cancel":"all"} */ - private HttpResponse cancelDeploy(String tenantName, String applicationName, String choice) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse cancelDeploy(String tenantName, String applicationName, String instanceName, String choice) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); StringBuilder response = new StringBuilder(); controller.applications().lockOrThrow(id, application -> { Change change = application.get().change(); @@ -1038,8 +1054,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return tenant(tenant.get(), request); } - private HttpResponse deleteApplication(String tenantName, String applicationName, HttpRequest request) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + private HttpResponse deleteApplication(String tenantName, String applicationName, String instanceName, HttpRequest request) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); Optional credentials = controller.tenants().require(id.tenant()).type() == Tenant.Type.user ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); @@ -1412,7 +1428,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new RunId(appIdFromPath(path), jobTypeFromPath(path), number); } - private HttpResponse submit(String tenant, String application, HttpRequest request) { + private HttpResponse submit(String tenant, String application, String instance, HttpRequest request) { Map dataParts = parseDataParts(request); Inspector submitOptions = SlimeUtils.jsonToSlime(dataParts.get(EnvironmentResource.SUBMIT_OPTIONS)).get(); SourceRevision sourceRevision = toSourceRevision(submitOptions); @@ -1427,6 +1443,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return JobControllerApiHandlerHelper.submitResponse(controller.jobController(), tenant, application, + instance, sourceRevision, authorEmail, projectId, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index ef7510ad18c..cd7c0d6236d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -416,10 +416,10 @@ class JobControllerApiHandlerHelper { * * @return Response with the new application version */ - static HttpResponse submitResponse(JobController jobController, String tenant, String application, + static HttpResponse submitResponse(JobController jobController, String tenant, String application, String instance, SourceRevision sourceRevision, String authorEmail, long projectId, ApplicationPackage applicationPackage, byte[] testPackage) { - ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, "default"), + ApplicationVersion version = jobController.submit(ApplicationId.from(tenant, application, instance), sourceRevision, authorEmail, projectId, @@ -444,8 +444,8 @@ class JobControllerApiHandlerHelper { } /** Unregisters the application from the internal deployment pipeline. */ - static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName) { - ApplicationId id = ApplicationId.from(tenantName, applicationName, "default"); + static HttpResponse unregisterResponse(JobController jobs, String tenantName, String applicationName, String instanceName) { + ApplicationId id = ApplicationId.from(tenantName, applicationName, instanceName); jobs.unregister(id); Slime slime = new Slime(); slime.setObject().setString("message", "Unregistered '" + id + "' from internal deployment pipeline."); -- cgit v1.2.3 From 8bef388ad6ed828aef1442f524ca6a92a2b36ea5 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Thu, 13 Jun 2019 13:06:42 +0200 Subject: Point to instance-specific paths --- .../controller/restapi/application/ApplicationApiHandler.java | 7 +++++-- .../restapi/application/responses/application-reference-2.json | 2 +- .../restapi/application/responses/application-reference.json | 2 +- .../restapi/application/responses/tenant-with-application.json | 2 +- .../controller/restapi/user/responses/application-created.json | 2 +- 5 files changed, 9 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 0f513854206..7ba78034839 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1274,8 +1274,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { object.setString("tenant", application.id().tenant().value()); object.setString("application", application.id().application().value()); object.setString("instance", application.id().instance().value()); - object.setString("url", withPath("/application/v4/tenant/" + application.id().tenant().value() + - "/application/" + application.id().application().value(), request.getUri()).toString()); + object.setString("url", withPath("/application/v4" + + "/tenant/" + application.id().tenant().value() + + "/application/" + application.id().application().value() + + "/instance/" + application.id().instance().value(), + request.getUri()).toString()); } private Slime toSlime(ActivateResult result) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json index ff22b95739d..0c9dc2ca1e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference-2.json @@ -2,5 +2,5 @@ "tenant": "tenant2", "application": "application2", "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2" + "url": "http://localhost:8080/application/v4/tenant/tenant2/application/application2/instance/default" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json index 1d56944f6bc..7117cc22507 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json @@ -2,5 +2,5 @@ "tenant": "tenant1", "application":"application1", "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1" + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index b222c33291c..0a99248b012 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -8,7 +8,7 @@ "tenant": "tenant1", "application":"application1", "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1" + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json index 2a779f0ee55..6cf4dc76173 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/responses/application-created.json @@ -2,5 +2,5 @@ "tenant": "my-tenant", "application": "my-app", "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app" + "url": "http://localhost:8080/application/v4/tenant/my-tenant/application/my-app/instance/default" } -- cgit v1.2.3 From c3fe12576146bde732a98495fb9b6fa87b6275e1 Mon Sep 17 00:00:00 2001 From: Harald Musum Date: Thu, 13 Jun 2019 14:23:53 +0200 Subject: Prepare for removing unused methods, change to be empty default methods --- .../api/integration/configserver/ConfigServer.java | 4 +- .../controller/integration/ConfigServerMock.java | 87 ++++++++-------------- 2 files changed, 34 insertions(+), 57 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 0c6bee4073d..68711e25958 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -26,8 +26,8 @@ public interface ConfigServer { interface PreparedApplication { // TODO: Remove the two methods below - void activate(); - List messages(); + default void activate() {} + default List messages() { return List.of(); } PrepareResponse prepareResponse(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index cb39e066e16..2b9296d21cf 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -237,62 +237,39 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer this.rotationNames.put(deployment, Set.copyOf(rotationNames)); - return new PreparedApplication() { - - // TODO: Remove when no longer part of interface - public void activate() {} - - // TODO: Remove when no longer part of interface - public List messages() { - Log warning = new Log(); - warning.level = "WARNING"; - warning.time = 1; - warning.message = "The warning"; - - Log info = new Log(); - info.level = "INFO"; - info.time = 2; - info.message = "The info"; - - return List.of(warning, info); + return () -> { + Application application = applications.get(deployment.applicationId()); + application.activate(); + List nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); + for (Node node : nodes) { + nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(), + Node.State.active, + node.type(), + node.owner(), + node.currentVersion(), + application.version().get())); } - - @Override - public PrepareResponse prepareResponse() { - Application application = applications.get(deployment.applicationId()); - application.activate(); - List nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId()); - for (Node node : nodes) { - nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(), - Node.State.active, - node.type(), - node.owner(), - node.currentVersion(), - application.version().get())); - } - serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(), - deployment.zoneId(), - false, - 2, - nodes.stream() - .map(node -> new ServiceConvergence.Status(node.hostname(), - 43, - "container", - 1)) - .collect(Collectors.toList()))); - - PrepareResponse prepareResponse = new PrepareResponse(); - prepareResponse.message = "foo"; - prepareResponse.configChangeActions = configChangeActions != null - ? configChangeActions - : new ConfigChangeActions(Collections.emptyList(), - Collections.emptyList()); - setConfigChangeActions(null); - prepareResponse.tenant = new TenantId("tenant"); - prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList()); - return prepareResponse; - } - + serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(), + deployment.zoneId(), + false, + 2, + nodes.stream() + .map(node -> new ServiceConvergence.Status(node.hostname(), + 43, + "container", + 1)) + .collect(Collectors.toList()))); + + PrepareResponse prepareResponse = new PrepareResponse(); + prepareResponse.message = "foo"; + prepareResponse.configChangeActions = configChangeActions != null + ? configChangeActions + : new ConfigChangeActions(Collections.emptyList(), + Collections.emptyList()); + setConfigChangeActions(null); + prepareResponse.tenant = new TenantId("tenant"); + prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList()); + return prepareResponse; }; } -- cgit v1.2.3 From 33c29c4faea7762e53cb5a6491ae446944c08e2a Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 13 Jun 2019 14:34:19 +0200 Subject: Add support for containerEndpoints deploy parameter --- .../api/integration/configserver/ConfigServer.java | 3 +- .../configserver/ContainerEndpoint.java | 59 ++++++++++++++++++++++ .../hosted/controller/ApplicationController.java | 3 +- .../controller/integration/ConfigServerMock.java | 4 +- 4 files changed, 65 insertions(+), 4 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 0c6bee4073d..cfd1ba8d9ed 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -31,7 +31,8 @@ public interface ConfigServer { PrepareResponse prepareResponse(); } - PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, byte[] content); + PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, + List containerEndpoints, byte[] content); void restart(DeploymentId deployment, Optional hostname); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java new file mode 100644 index 00000000000..2134320bdc1 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java @@ -0,0 +1,59 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.configserver; + +import java.util.List; +import java.util.Objects; + +/** + * This represents a list of one or more names for a container cluster. + * + * @author mpolden + */ +public class ContainerEndpoint { + + private final String clusterId; + private final List names; + + public ContainerEndpoint(String clusterId, List names) { + this.clusterId = nonEmpty(clusterId, "message must be non-empty"); + this.names = List.copyOf(Objects.requireNonNull(names, "names must be non-null")); + } + + /** ID of the cluster to which this points */ + public String clusterId() { + return clusterId; + } + + /** + * All valid DNS names for this endpoint. This can contain both proper DNS names and synthetic identifiers used for + * routing, such as a Host header value that is not necessarily a proper DNS name. + */ + public List names() { + return names; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContainerEndpoint that = (ContainerEndpoint) o; + return clusterId.equals(that.clusterId) && + names.equals(that.names); + } + + @Override + public int hashCode() { + return Objects.hash(clusterId, names); + } + + @Override + public String toString() { + return "container endpoint for " + clusterId + " " + names; + } + + private static String nonEmpty(String s, String message) { + if (s == null || s.isBlank()) throw new IllegalArgumentException(message); + return s; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c1a92ca0abd..0b9349b67a9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -90,7 +90,6 @@ import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.function.Consumer; -import java.util.function.Predicate; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -426,7 +425,7 @@ public class ApplicationController { Set rotationNames) { DeploymentId deploymentId = new DeploymentId(application, zone); ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, rotationNames, applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationPackage.zippedContent()); // Refresh routing policies on successful deployment. At this point we can safely assume that the config server // has allocated load balancers for the deployment. diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index cb39e066e16..f70154d8f8c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -223,7 +224,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer } @Override - public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, byte[] content) { + public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, + List containerEndpoints, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; -- cgit v1.2.3 From 2b827617d7c8ed1fea0e06b4786c601764bf0b41 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 13 Jun 2019 16:02:48 +0200 Subject: New endpoints not yet ready --- .../com/yahoo/vespa/hosted/controller/ApplicationController.java | 9 ++++----- .../vespa/hosted/controller/maintenance/RoutingPoliciesTest.java | 1 + 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c1a92ca0abd..23f263ce255 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -550,13 +550,12 @@ public class ApplicationController { throw new NotExistsException("Deployment", id.toString()); try { - return Optional.of(routingPolicies.get(id)) - .filter(policies -> ! policies.isEmpty()) - .map(policies -> policies.stream() + return Optional.of(routingGenerator.clusterEndpoints(id)) + .filter(endpoints -> ! endpoints.isEmpty()) + .orElseGet(() -> routingPolicies.get(id).stream() .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), - policy -> policy.endpointIn(controller.system()).url()))) - .orElseGet(() -> routingGenerator.clusterEndpoints(id)); + policy -> policy.endpointIn(controller.system()).url()))); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java index f19c309dd67..449ca509ee4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java @@ -202,6 +202,7 @@ public class RoutingPoliciesTest { public void cluster_endpoints_resolve_from_policies() { provisionLoadBalancers(3, app1.id(), zone1); tester.deployCompletely(app1, applicationPackage); + tester.controllerTester().routingGenerator().putEndpoints(new DeploymentId(app1.id(), zone1), Collections.emptyList()); assertEquals(Map.of(ClusterSpec.Id.from("c0"), URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), ClusterSpec.Id.from("c1"), -- cgit v1.2.3 From 6876f42d547e0c650471155097a0be027f88a52d Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Fri, 14 Jun 2019 01:17:55 +0200 Subject: Remove cloud from ZoneId --- .../java/com/yahoo/config/provision/zone/ZoneId.java | 20 +------------------- .../java/com/yahoo/config/provision/ZoneIdTest.java | 3 --- .../deployment/InternalStepRunnerTest.java | 2 +- .../controller/integration/ZoneFilterMock.java | 18 +++++++++--------- .../restapi/application/ApplicationApiTest.java | 4 ++-- .../responses/delete-with-active-deployments.json | 2 +- .../responses/us-east-3-log-without-first.json | 2 +- 7 files changed, 15 insertions(+), 36 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index c6aaf05492e..5e664e00b4c 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -18,17 +18,13 @@ import java.util.Objects; public class ZoneId { // TODO: Replace usages of environment + region with usages of this. - // TODO: Remove static factory methods not specifying cloud and system - private final Environment environment; private final RegionName region; - private final CloudName cloud; private final SystemName system; private ZoneId(Environment environment, RegionName region, CloudName cloud, SystemName system) { this.environment = Objects.requireNonNull(environment, "environment must be non-null"); this.region = Objects.requireNonNull(region, "region must be non-null"); - this.cloud = Objects.requireNonNull(cloud, "cloud must be non-null"); this.system = Objects.requireNonNull(system, "system must be non-null"); } @@ -69,10 +65,6 @@ public class ZoneId { return new ZoneId(environment, region, cloud, SystemName.defaultSystem()); } - public static ZoneId from(String environment, String region, String cloud) { - return new ZoneId(Environment.from(environment), RegionName.from(region), CloudName.from(cloud), SystemName.defaultSystem()); - } - public static ZoneId from(String environment, String region, String cloud, String system) { return new ZoneId(Environment.from(environment), RegionName.from(region), CloudName.from(cloud), SystemName.from(system)); } @@ -89,10 +81,6 @@ public class ZoneId { return region; } - public CloudName cloud() { - return cloud; - } - public SystemName system() { return system; } @@ -100,20 +88,14 @@ public class ZoneId { /** Returns the serialised value of this. Inverse of {@code ZoneId.from(String value)}. */ public String value() { return environment + "." + region; - // TODO: Change to the below when there only methods use constructor including cloud and system are used and - // all serialized values contain cloud and system - // return cloud + "." + system + "." + environment + "." + region; } @Override public String toString() { - return "zone " + value() + " in " + cloud; - // TODO: Use the below (need to fix some use of toString() in tests first) - //return "zone " + cloud + "." + system + "." + environment + "." + region; + return value(); } @Override - // TODO: Update to check cloud and system when everyone use methods that specify cloud and system public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java index 83f8412f66b..27d45ba7d7d 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java @@ -26,19 +26,16 @@ public class ZoneIdTest { ZoneId zoneId = ZoneId.from(environment, region); assertEquals(region, zoneId.region()); assertEquals(environment, zoneId.environment()); - assertEquals(CloudName.defaultName(), zoneId.cloud()); assertEquals(SystemName.defaultSystem(), zoneId.system()); ZoneId zoneIdWithSystem = ZoneId.from(system, environment, region); assertEquals(region, zoneIdWithSystem.region()); assertEquals(environment, zoneIdWithSystem.environment()); - assertEquals(CloudName.defaultName(), zoneIdWithSystem.cloud()); assertEquals(system, zoneIdWithSystem.system()); ZoneId zoneIdWithCloudAndSystem = ZoneId.from(environment, region, cloud, system); assertEquals(region, zoneIdWithCloudAndSystem.region()); assertEquals(environment, zoneIdWithCloudAndSystem.environment()); - assertEquals(cloud, zoneIdWithCloudAndSystem.cloud()); assertEquals(system, zoneIdWithCloudAndSystem.system()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 4fb6a2e34fb..095651df033 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -374,7 +374,7 @@ public class InternalStepRunnerTest { tester.runner().run(); assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests)); assertTestLogEntries(id, Step.copyVespaLogs, - new LogEntry(lastId + 2, tester.clock().millis(), debug, "Copying Vespa log from nodes of tenant.application in zone test.us-east-1 in default ..."), + new LogEntry(lastId + 2, tester.clock().millis(), debug, "Copying Vespa log from nodes of tenant.application in test.us-east-1 ..."), new LogEntry(lastId + 3, 1554970337084L, info, "17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\n" + "Switching to the latest deployed set of configurations and components. Application switch number: 2"), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 5cf8268c18e..700e6e9cb42 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -43,7 +43,7 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList all() { - return filter(zoneId -> true); + return filter(zone -> true); } @Override @@ -63,17 +63,17 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList in(Environment... environments) { - return filter(zoneId -> new HashSet<>(Arrays.asList(environments)).contains(zoneId.environment())); + return filter(zone -> new HashSet<>(Arrays.asList(environments)).contains(zone.getEnvironment())); } @Override public ZoneList in(RegionName... regions) { - return filter(zoneId -> new HashSet<>(Arrays.asList(regions)).contains(zoneId.region())); + return filter(zone -> new HashSet<>(Arrays.asList(regions)).contains(zone.getRegionName())); } @Override public ZoneList among(ZoneId... zones) { - return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId)); + return filter(zone -> new HashSet<>(Arrays.asList(zones)).contains(zone.getId())); } @Override @@ -88,15 +88,15 @@ public class ZoneFilterMock implements ZoneList { @Override public ZoneList ofCloud(CloudName cloud) { - return filter(zoneId -> zoneId.cloud().equals(cloud)); + return filter(zone -> zone.getCloudName().equals(cloud)); } - private ZoneFilterMock filter(Predicate condition) { + private ZoneFilterMock filter(Predicate condition) { return new ZoneFilterMock( zones.stream() - .filter(zoneApi -> negate ? - condition.negate().test(zoneApi.toDeprecatedId()) : - condition.test(zoneApi.toDeprecatedId())) + .filter(zone -> negate ? + condition.negate().test(zone) : + condition.test(zone)) .collect(Collectors.toList()), false); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 632fea8cb1f..99ea9cc02d0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -719,14 +719,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // Invalid deployment fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in zone prod.us-east-3 in default\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in prod.us-east-3\"}", 404); // Change status of non-existing deployment fails tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in zone prod.us-east-3 in default\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in prod.us-east-3\"}", 404); // GET global rotation status diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json index 012b0031489..25fed881dec 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-east-1 in default, zone prod.us-central-1 in default" + "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: dev.us-east-1, prod.us-central-1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json index 77d5b3479be..390024fe33d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json @@ -12,7 +12,7 @@ { "at": 1000, "type": "debug", - "message": "Deactivating tester of tenant.application in zone prod.us-east-3 in default ..." + "message": "Deactivating tester of tenant.application in prod.us-east-3 ..." } ] }, -- cgit v1.2.3 From 86b43a2702b145639a337bd5588eb6f355d91b13 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 14 Jun 2019 11:27:14 +0200 Subject: Use a single lock when refreshing routing policies --- .../controller/maintenance/RoutingPolicies.java | 109 ++++++++++----------- 1 file changed, 52 insertions(+), 57 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java index 508401b0e14..4a98cb49227 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -74,46 +75,44 @@ public class RoutingPolicies { if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; var lbs = new LoadBalancers(application, zone, controller.applications().configServer() .getLoadBalancers(application, zone)); - removeObsoleteEndpointsFromDns(lbs); - storePoliciesOf(lbs); - removeObsoletePolicies(lbs); - registerEndpointsInDns(lbs); + try (var lock = db.lockRoutingPolicies()) { + removeObsoleteEndpointsFromDns(lbs, lock); + storePoliciesOf(lbs, lock); + removeObsoletePolicies(lbs, lock); + registerEndpointsInDns(lbs, lock); + } } /** Create global endpoints for given route, if any */ - private void registerEndpointsInDns(LoadBalancers loadBalancers) { - try (var lock = db.lockRoutingPolicies()) { - Map> routingTable = routingTableFrom(get(loadBalancers.application)); - - // Create DNS record for each routing ID - for (Map.Entry> routeEntry : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().rotation(), - controller.system()); - Set targets = routeEntry.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } + private void registerEndpointsInDns(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + Map> routingTable = routingTableFrom(get(loadBalancers.application)); + + // Create DNS record for each routing ID + for (Map.Entry> routeEntry : routingTable.entrySet()) { + Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().rotation(), + controller.system()); + Set targets = routeEntry.getValue() + .stream() + .filter(policy -> policy.dnsZone().isPresent()) + .map(policy -> new AliasTarget(policy.canonicalName(), + policy.dnsZone().get(), + policy.zone())) + .collect(Collectors.toSet()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); } } /** Store routing policies for given route */ - private void storePoliciesOf(LoadBalancers loadBalancers) { - try (var lock = db.lockRoutingPolicies()) { - Set policies = new LinkedHashSet<>(get(loadBalancers.application)); - for (LoadBalancer loadBalancer : loadBalancers.list) { - RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer); - if (!policies.add(policy)) { - policies.remove(policy); - policies.add(policy); - } + private void storePoliciesOf(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + Set policies = new LinkedHashSet<>(get(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer); + if (!policies.add(policy)) { + policies.remove(policy); + policies.add(policy); } - db.writeRoutingPolicies(loadBalancers.application, policies); } + db.writeRoutingPolicies(loadBalancers.application, policies); } /** Create a policy for given load balancer and register a CNAME for it */ @@ -128,36 +127,32 @@ public class RoutingPolicies { } /** Remove obsolete policies for given route and their CNAME records */ - private void removeObsoletePolicies(LoadBalancers loadBalancers) { - try (var lock = db.lockRoutingPolicies()) { - var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); - var removalCandidates = new HashSet<>(allPolicies); - var activeLoadBalancers = loadBalancers.list.stream() - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Remove active load balancers and irrelevant zones from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || - !policy.zone().equals(loadBalancers.zone)); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller.system()).dnsName(); - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - allPolicies.remove(policy); - } - db.writeRoutingPolicies(loadBalancers.application, allPolicies); + private void removeObsoletePolicies(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); + var removalCandidates = new HashSet<>(allPolicies); + var activeLoadBalancers = loadBalancers.list.stream() + .map(LoadBalancer::hostname) + .collect(Collectors.toSet()); + // Remove active load balancers and irrelevant zones from candidates + removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || + !policy.zone().equals(loadBalancers.zone)); + for (var policy : removalCandidates) { + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + allPolicies.remove(policy); } + db.writeRoutingPolicies(loadBalancers.application, allPolicies); } /** Remove unreferenced global endpoints for given route from DNS */ - private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers) { - try (var lock = db.lockRoutingPolicies()) { - var zonePolicies = get(loadBalancers.application, loadBalancers.zone); - var removalCandidates = routingTableFrom(zonePolicies).keySet(); - var activeRoutingIds = routingIdsFrom(loadBalancers.list); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller.system()); - controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } + private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone); + var removalCandidates = routingTableFrom(zonePolicies).keySet(); + var activeRoutingIds = routingIdsFrom(loadBalancers.list); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); } } -- cgit v1.2.3 From 26d483792c6332d3a248b3f16f23c46f9635ce59 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 14 Jun 2019 11:28:29 +0200 Subject: Log when name service request is queued --- .../java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java | 1 + 1 file changed, 1 insertion(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java index 299ea168c7a..4e461534cc0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java @@ -78,6 +78,7 @@ public class NameServiceForwarder { "requests. This likely means that the name service is not successfully " + "executing requests"); } + log.log(LogLevel.INFO, "Queueing name service request: " + request); db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests)); } } -- cgit v1.2.3 From 7e2d577daf548e171a7d7bf16a6996f9b894c330 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Fri, 14 Jun 2019 12:55:49 +0200 Subject: Support paths with instance after application in application/v4 --- .../hosted/controller/api/role/PathGroup.java | 31 ++- .../hosted/controller/ApplicationController.java | 2 +- .../restapi/application/ApplicationApiHandler.java | 30 ++- .../restapi/ContainerControllerTester.java | 7 +- .../restapi/application/ApplicationApiTest.java | 267 ++++++++++----------- .../application/responses/application-nodes.json | 2 +- .../responses/application-reference.json | 4 +- ...cation-without-change-multiple-deployments.json | 12 +- .../restapi/application/responses/application.json | 12 +- .../responses/application1-recursive.json | 4 +- .../responses/delete-with-active-deployments.json | 2 +- .../responses/deployment-job-accepted.json | 2 +- .../restapi/application/responses/deployment.json | 6 +- .../application/responses/dev-us-west-1.json | 6 +- .../responses/global-rotation-delete.json | 2 +- .../application/responses/global-rotation-put.json | 2 +- .../restapi/application/responses/jobs.json | 10 +- .../application/responses/prod-us-central-1.json | 6 +- .../restapi/application/responses/services.json | 4 +- .../application/responses/system-test-job.json | 2 +- .../responses/tenant-with-application.json | 4 +- .../restapi/deployment/BadgeApiTest.java | 2 +- .../restapi/deployment/DeploymentApiTest.java | 9 +- 23 files changed, 230 insertions(+), 198 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index ec44b5605b6..f4816062e43 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -44,13 +44,15 @@ enum PathGroup { /** Paths used by tenant administrators. */ tenantInfo(Matcher.tenant, Optional.of("/api"), - "/application/v4/tenant/{tenant}/application/"), + "/application/v4/tenant/{tenant}/application/", + "/application/v4/tenant/{tenant}/application/{application}/instance/"), /** Path for the base application resource. */ application(Matcher.tenant, Matcher.application, Optional.of("/api"), - "/application/v4/tenant/{tenant}/application/{application}"), + "/application/v4/tenant/{tenant}/application/{application}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}"), /** Paths used for user management on the application level. */ applicationUsers(Matcher.tenant, @@ -63,7 +65,13 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/deploying/{*}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{*}", "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{*}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/nodes", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/logs", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{*}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/{*}", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended", @@ -74,6 +82,8 @@ enum PathGroup { developmentRestart(Matcher.tenant, Matcher.application, Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/dev/region/{region}/restart", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/perf/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{ignored}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{ignored}/restart"), @@ -81,6 +91,9 @@ enum PathGroup { productionRestart(Matcher.tenant, Matcher.application, Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/prod/region/{region}/restart", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/test/region/{region}/restart", + "/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/environment/staging/region/{region}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{ignored}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{ignored}/restart", "/application/v4/tenant/{tenant}/application/{application}/environment/staging/region/{region}/instance/{ignored}/restart"), @@ -90,6 +103,10 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploy/{job}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/dev/region/{region}/deploy", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/perf/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/perf/region/{region}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/dev/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/perf/region/{region}/instance/{instance}", @@ -99,6 +116,12 @@ enum PathGroup { productionDeployment(Matcher.tenant, Matcher.application, Optional.of("/api"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/prod/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/prod/region/{region}/deploy", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/test/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/test/region/{region}/deploy", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/staging/region/{region}", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/staging/region/{region}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}", "/application/v4/tenant/{tenant}/application/{application}/environment/prod/region/{region}/instance/{instance}/deploy", "/application/v4/tenant/{tenant}/application/{application}/environment/test/region/{region}/instance/{instance}", @@ -110,13 +133,15 @@ enum PathGroup { submission(Matcher.tenant, Matcher.application, Optional.of("/api"), - "/application/v4/tenant/{tenant}/application/{application}/submit"), + "/application/v4/tenant/{tenant}/application/{application}/submit", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit"), /** Paths used for other tasks by build services. */ // TODO: This will vanish. buildService(Matcher.tenant, Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport", "/application/v4/tenant/{tenant}/application/{application}/promote", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 4e6c45998ec..c3d5788ccb3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -246,7 +246,7 @@ public class ApplicationController { if (credentials.isEmpty()) throw new IllegalArgumentException("Could not create '" + id + "': No credentials provided"); - if (id.instance().isDefault()) // Only store the application permits for non-user applications. + if ( ! id.instance().isTester()) // Only store the application permits for non-user applications. accessControl.createApplication(id, credentials.get()); } LockedApplication application = new LockedApplication(new Application(id, clock.instant()), lock); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 7ba78034839..b4483010176 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -187,6 +187,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/service/{service}/{*}")) return service(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), path.get("service"), path.getRest(), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation")) return rotationStatus(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return getGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service")) return services(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -199,8 +207,8 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePUT(Path path, HttpRequest request) { if (path.matches("/application/v4/user")) return createUser(request); if (path.matches("/application/v4/tenant/{tenant}")) return updateTenant(path.get("tenant"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), false, request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -218,9 +226,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), path.get("instance"), true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), path.get("instance"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{ignored}/jobreport")) return notifyJobCompletion(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return submit(path.get("tenant"), path.get("application"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return trigger(appIdFromPath(path), jobTypeFromPath(path), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/pause")) return pause(appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); @@ -245,9 +258,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/{choice}")) return cancelDeploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("choice")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/submit")) return JobControllerApiHandlerHelper.unregisterResponse(controller.jobController(), path.get("tenant"), path.get("application"), path.get("instance")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.abortJobResponse(controller.jobController(), appIdFromPath(path), jobTypeFromPath(path)); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deactivate(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) - return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/override")) return setGlobalRotationOverride(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), true, request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -545,7 +559,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { deploymentObject.setString("url", withPath(request.getUri().getPath() + "/environment/" + deployment.zone().environment().value() + "/region/" + deployment.zone().region().value() + - "/instance/" + application.id().instance().value(), + ( request.getUri().getPath().contains("/instance/") ? "" : "/instance/" + application.id().instance().value()), request.getUri()).toString()); } } @@ -1072,9 +1086,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // TODO: Change to return JSON return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, ApplicationResource.API_PATH, applicationName, + "instance", instanceName, EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); + "region", region)); } /** @@ -1185,7 +1199,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } Cursor applicationArray = object.setArray("applications"); for (Application application : controller.applications().asList(tenant.name())) { - if (application.id().instance().isDefault()) {// TODO: Skip non-default applications until supported properly + if ( ! application.id().instance().isTester()) { if (recurseOverApplications(request)) toSlime(applicationArray.addObject(), application, request); else diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index c851cb18e8c..76c505ff8f8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -72,11 +72,10 @@ public class ContainerControllerTester { public ContainerTester containerTester() { return containerTester; } public Application createApplication() { - return createApplication("domain1","tenant1", - "application1"); + return createApplication("domain1","tenant1", "application1", "default"); } - public Application createApplication(String athensDomain, String tenant, String application) { + public Application createApplication(String athensDomain, String tenant, String application, String instance) { AthenzDomain domain1 = addTenantAthenzDomain(athensDomain, "user"); AthenzPrincipal user = new AthenzPrincipal(new AthenzUser("user")); AthenzCredentials credentials = new AthenzCredentials(user, domain1, new OktaAccessToken("okta-token")); @@ -86,7 +85,7 @@ public class ContainerControllerTester { Optional.of(new PropertyId("1234"))); controller().tenants().create(tenantSpec, credentials); - ApplicationId app = ApplicationId.from(tenant, application, "default"); + ApplicationId app = ApplicationId.from(tenant, application, instance); return controller().applications().createApplication(app, Optional.of(credentials)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 99bc07a9528..2c17abf9e90 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -193,7 +193,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("tenant-with-contact-info.json")); // POST (create) an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); @@ -204,12 +204,15 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET tenant applications tester.assertResponse(request("/application/v4/tenant/tenant1/application/", GET).userIdentity(USER_ID), new File("application-list.json")); + // GET tenant applications (instances of "application1" only) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/", GET).userIdentity(USER_ID), + new File("application-list.json")); addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR)); // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification) MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data))) .userIdentity(USER_ID), @@ -217,7 +220,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application to a zone. This simulates calls done by our tenant pipeline. - ApplicationId id = ApplicationId.from("tenant1", "application1", "default"); + ApplicationId id = ApplicationId.from("tenant1", "application1", "instance1"); long screwdriverProjectId = 123; addScrewdriverUserToDeployRole(SCREWDRIVER_ID, @@ -232,13 +235,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // ... systemtest - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1/", POST) .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/test/region/us-east-1"); controllerTester.jobCompletion(JobType.systemTest) .application(id) @@ -246,20 +249,20 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // ... staging - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1/", POST) .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/staging/region/us-east-3"); controllerTester.jobCompletion(JobType.stagingTest) .application(id) .projectId(screwdriverProjectId) .submit(); // ... prod zone - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) .data(createApplicationDeployData(Optional.empty(), false)) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -274,10 +277,10 @@ public class ApplicationApiTest extends ControllerContainerTest { Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber - 1)), true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application package found for tenant1.application1 with version 1.0.41-commit1\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"No application package found for tenant1.application1.instance1 with version 1.0.41-commit1\"}", 400); // POST an application deployment to a production zone - operator emergency deployment - works with known package @@ -285,7 +288,7 @@ public class ApplicationApiTest extends ControllerContainerTest { Optional.of(ApplicationVersion.from(BuildJob.defaultSourceRevision, BuildJob.defaultBuildNumber)), true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/", POST) .data(entity) .userIdentity(HOSTED_VESPA_OPERATOR), new File("deploy-result.json")); @@ -368,15 +371,15 @@ public class ApplicationApiTest extends ControllerContainerTest { setDeploymentMaintainedInfo(controllerTester); // GET tenant application deployments - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) .userIdentity(USER_ID), new File("application.json")); // GET an application deployment - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", GET) .userIdentity(USER_ID), new File("deployment.json")); - addIssues(controllerTester, ApplicationId.from("tenant1", "application1", "default")); + addIssues(controllerTester, ApplicationId.from("tenant1", "application1", "instance1")); // GET at root, with "&recursive=deployment", returns info about all tenants, their applications and their deployments tester.assertResponse(request("/application/v4/", GET) .userIdentity(USER_ID) @@ -393,13 +396,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .recursive("true"), new File("tenant1-recursive.json")); // GET at an application, with "&recursive=true", returns full info about its deployments - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) .userIdentity(USER_ID) .recursive("true"), new File("application1-recursive.json")); // GET nodes - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/nodes", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/nodes", GET) .userIdentity(USER_ID), new File("application-nodes.json")); @@ -409,143 +412,143 @@ public class ApplicationApiTest extends ControllerContainerTest { "INFO - All good"); // DELETE (cancel) ongoing change - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1'\"}"); + "{\"message\":\"Changed deployment from 'application change to 1.0.42-commit1' to 'no change' for application 'tenant1.application1.instance1'\"}"); // DELETE (cancel) again is a no-op - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", DELETE) .userIdentity(USER_ID) .data("{\"cancel\":\"all\"}"), - "{\"message\":\"No deployment in progress for application 'tenant1.application1' at this time\"}"); + "{\"message\":\"No deployment in progress for application 'tenant1.application1.instance1' at this time\"}"); // POST pinning to a given version to an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1.0"), - "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); assertTrue("Action is logged to audit log", tester.controller().auditLogger().readLog().entries().stream() - .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/deploying/pin"))); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + .anyMatch(entry -> entry.resource().equals("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin"))); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); // DELETE only the pin to a given version - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1'\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'upgrade to 6.1' for application 'tenant1.application1.instance1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":false}"); // POST pinning again - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", POST) .userIdentity(USER_ID) .data("6.1"), - "{\"message\":\"Triggered pin to 6.1 for tenant1.application1\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Triggered pin to 6.1 for tenant1.application1.instance1\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"platform\":\"6.1\",\"pinned\":true}"); // DELETE only the version, but leave the pin - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/platform", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/platform", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1'\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Changed deployment from 'pin to 6.1' to 'pin to current platform' for application 'tenant1.application1.instance1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{\"pinned\":true}"); // DELETE also the pin to a given version - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying/pin", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying/pin", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1'\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/deploying", GET) + "{\"message\":\"Changed deployment from 'pin to current platform' to 'no change' for application 'tenant1.application1.instance1'\"}"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploying", GET) .userIdentity(USER_ID), "{}"); // POST a pause to a production job - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1/pause", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1/pause", POST) .userIdentity(USER_ID), - "{\"message\":\"production-us-west-1 for tenant1.application1 paused for " + DeploymentTrigger.maxPause + "\"}"); + "{\"message\":\"production-us-west-1 for tenant1.application1.instance1 paused for " + DeploymentTrigger.maxPause + "\"}"); // POST a triggering to the same production job - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1", POST) .userIdentity(USER_ID), - "{\"message\":\"Triggered production-us-west-1 for tenant1.application1\"}"); + "{\"message\":\"Triggered production-us-west-1 for tenant1.application1.instance1\"}"); // POST a 'restart application' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); // POST a 'restart application' command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); // POST a 'restart application' in staging environment command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1"); // POST a 'restart application' in staging test command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1"); // POST a 'restart application' in staging dev command - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/default/restart", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/default"); + "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1"); // POST a 'restart application' command with a host filter (other filters not supported yet) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/restart?hostname=host1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=host1", POST) .screwdriverIdentity(SCREWDRIVER_ID), "{\"error-code\":\"INTERNAL_SERVER_ERROR\",\"message\":\"No node with the hostname host1 is known.\"}", 500); // GET suspended - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/suspended", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/suspended", GET) .userIdentity(USER_ID), new File("suspended.json")); // GET services - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service", GET) .userIdentity(USER_ID), new File("services.json")); // GET service - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", GET) .userIdentity(USER_ID), new File("service.json")); // DELETE application with active deployments fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("delete-with-active-deployments.json"), 400); // DELETE (deactivate) a deployment - dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1"); // DELETE (deactivate) a deployment - prod - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); // DELETE (deactivate) a deployment is idempotent - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default"); + "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); // POST an application package to start a deployment to dev - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/deploy/dev-us-east-1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1", POST) .userIdentity(USER_ID) .data(createApplicationDeployData(applicationPackage, false)), new File("deployment-job-accepted.json")); // POST an application package and a test jar, submitting a new application for internal pipeline deployment. // First attempt does not have an Athenz service definition in deployment spec, and is accepted. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(applicationPackage)), "{\"message\":\"Application package version: 1.0.43-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); @@ -556,7 +559,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN_2.getName()), AthenzService.from("service")) .region("us-west-1") .build(); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithServiceForWrongDomain)), "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Athenz domain in deployment.xml: [domain2] must match tenant domain: [domain1]\"}", 400); @@ -567,13 +570,13 @@ public class ApplicationApiTest extends ControllerContainerTest { .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from(ATHENZ_TENANT_DOMAIN.getName()), AthenzService.from("service")) .region("us-west-1") .build(); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(createApplicationSubmissionData(packageWithService)), "{\"message\":\"Application package version: 1.0.44-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); // Fourth attempt has a wrong content hash in a header, and fails. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .header("X-Content-Hash", "not/the/right/hash") .data(createApplicationSubmissionData(packageWithService)), @@ -581,14 +584,14 @@ public class ApplicationApiTest extends ControllerContainerTest { // Fifth attempt has the right content hash in a header, and succeeds. MultiPartStreamer streamer = createApplicationSubmissionData(packageWithService); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", POST) .screwdriverIdentity(SCREWDRIVER_ID) .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(streamer::data))) .data(streamer), "{\"message\":\"Application package version: 1.0.45-d00d, source revision of repository 'repo', branch 'master' with commit 'd00d', by a@b, built against 6.1 at 1970-01-01T00:00:01Z\"}"); - ApplicationId app1 = ApplicationId.from("tenant1", "application1", "default"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) + ApplicationId app1 = ApplicationId.from("tenant1", "application1", "instance1"); + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/jobreport", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(asJson(DeploymentJobs.JobReport.ofComponent(app1, 1234, @@ -602,31 +605,31 @@ public class ApplicationApiTest extends ControllerContainerTest { // GET deployment job overview, after triggering system and staging test jobs. assertEquals(2, tester.controller().applications().deploymentTrigger().triggerReadyJobs()); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job", GET) .userIdentity(USER_ID), new File("jobs.json")); // GET system test job overview. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test", GET) .userIdentity(USER_ID), new File("system-test-job.json")); // GET system test run 1 details. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test/run/1", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1", GET) .userIdentity(USER_ID), new File("system-test-details.json")); // DELETE a running job to have it aborted. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test", DELETE) .userIdentity(USER_ID), - "{\"message\":\"Aborting run 1 of staging-test for tenant1.application1\"}"); + "{\"message\":\"Aborting run 1 of staging-test for tenant1.application1.instance1\"}"); // DELETE submission to unsubscribe from continuous deployment. - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/submit", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/submit", DELETE) .userIdentity(HOSTED_VESPA_OPERATOR), - "{\"message\":\"Unregistered 'tenant1.application1' from internal deployment pipeline.\"}"); + "{\"message\":\"Unregistered 'tenant1.application1.instance1' from internal deployment pipeline.\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/jobreport", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/jobreport", POST) .screwdriverIdentity(SCREWDRIVER_ID) .data(asJson(DeploymentJobs.JobReport.ofComponent(app1, 1234, @@ -652,16 +655,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .userIdentity(USER_ID), ""); - // Promote from pipeline - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/promote", POST) - .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"Successfully copied environment hosted-verified-prod to hosted-instance_tenant1_application1_placeholder_component_default\"}"); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/promote", POST) - .screwdriverIdentity(SCREWDRIVER_ID), - "{\"message\":\"Successfully copied environment hosted-instance_tenant1_application1_placeholder_component_default to hosted-instance_tenant1_application1_us-west-1_prod_default\"}"); - // DELETE an application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE).userIdentity(USER_ID) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), ""); // DELETE a tenant @@ -695,7 +690,7 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-west-1 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -712,37 +707,37 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // Invalid deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation", GET) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in zone prod.us-east-3 in default\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in zone prod.us-east-3 in default\"}", 404); // Change status of non-existing deployment fails - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/global-rotation/override", PUT) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1' has no deployment in zone prod.us-east-3 in default\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in zone prod.us-east-3 in default\"}", 404); // GET global rotation status setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation", GET) .userIdentity(USER_ID), new File("global-rotation.json")); // GET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", GET) .userIdentity(USER_ID), new File("global-rotation-get.json")); // SET global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", PUT) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", PUT) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-put.json")); // DELETE global rotation override status - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/global-rotation/override", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", DELETE) .userIdentity(USER_ID) .data("{\"reason\":\"unit-test\"}"), new File("global-rotation-delete.json")); @@ -762,7 +757,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("tenant-without-applications.json")); // Create application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); @@ -774,7 +769,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application to a prod zone - allowed when project ID is not specified MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/deploy", POST) .data(entity) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -793,7 +788,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); // POST (deploy) a system application without an application package - tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/proxy-host/environment/prod/region/us-central-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/hosted-vespa/application/proxy-host/environment/prod/region/us-central-1/instance/instance1/deploy", POST) .data(noAppEntity) .userIdentity(HOSTED_VESPA_OPERATOR), new File("deploy-no-deployment.json"), 400); @@ -813,7 +808,7 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 100); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -832,7 +827,7 @@ public class ApplicationApiTest extends ControllerContainerTest { startAndTestChange(controllerTester, id, projectId, applicationPackage, deployData, 101); // us-west-1 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -844,7 +839,7 @@ public class ApplicationApiTest extends ControllerContainerTest { setZoneInRotation("rotation-fqdn-1", ZoneId.from("prod", "us-west-1")); // us-east-3 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/deploy", POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), new File("deploy-result.json")); @@ -854,7 +849,7 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); setDeploymentMaintainedInfo(controllerTester); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", GET) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) .userIdentity(USER_ID), new File("application-without-change-multiple-deployments.json")); } @@ -938,16 +933,16 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // POST (create) an (empty) application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); // Create the same application again - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1': Application already exists\"}", + "{\"error-code\":\"BAD_REQUEST\",\"message\":\"Could not create 'tenant1.application1.instance1': Application already exists\"}", 400); ConfigServerMock configServer = (ConfigServerMock) container.components().getComponent(ConfigServerMock.class.getName()); @@ -955,28 +950,28 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST (deploy) an application with an invalid application package MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-failure.json"), 400); // POST (deploy) an application without available capacity configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to prepare application", ConfigServerException.ErrorCode.OUT_OF_CAPACITY, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-out-of-capacity.json"), 400); // POST (deploy) an application where activation fails configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Failed to activate application", ConfigServerException.ErrorCode.ACTIVATION_CONFLICT, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-activation-conflict.json"), 409); // POST (deploy) an application where we get an internal server error configServer.throwOnNextPrepare(new ConfigServerException(new URI("server-url"), "Internal server error", ConfigServerException.ErrorCode.INTERNAL_SERVER_ERROR, null)); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1/deploy", POST) .data(entity) .userIdentity(USER_ID), new File("deploy-internal-server-error.json"), 500); @@ -989,15 +984,15 @@ public class ApplicationApiTest extends ControllerContainerTest { 400); // DELETE application - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), ""); // DELETE application again - should produce 404 - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) .userIdentity(USER_ID), - "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1': Application not found\"}", + "{\"error-code\":\"NOT_FOUND\",\"message\":\"Could not delete application 'tenant1.application1.instance1': Application not found\"}", 404); // DELETE tenant @@ -1011,12 +1006,6 @@ public class ApplicationApiTest extends ControllerContainerTest { "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); - // Promote application chef env for nonexistent tenant/application - tester.assertResponse(request("/application/v4/tenant/dontexist/application/dontexist/environment/prod/region/us-west-1/instance/default/promote", POST) - .screwdriverIdentity(SCREWDRIVER_ID), - "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", - 403); - // Create legancy tenant name containing underscores tester.controller().curator().writeTenant(new AthenzTenant(TenantName.from("my_tenant"), ATHENZ_TENANT_DOMAIN, new Property("property1"), Optional.empty(), Optional.empty())); @@ -1066,14 +1055,14 @@ public class ApplicationApiTest extends ControllerContainerTest { 200); // Creating an application for an Athens domain the user is not admin for is disallowed - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(unauthorizedUser) .oktaAccessToken(OKTA_AT), "{\n \"code\" : 403,\n \"message\" : \"Access denied\"\n}", 403); // (Create it with the right tenant id) - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT), new File("application-reference.json"), @@ -1134,7 +1123,7 @@ public class ApplicationApiTest extends ControllerContainerTest { long screwdriverProjectId = 123; createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1"); + Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1", "default"); ScrewdriverId screwdriverId = new ScrewdriverId(Long.toString(screwdriverProjectId)); controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); @@ -1165,7 +1154,7 @@ public class ApplicationApiTest extends ControllerContainerTest { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1"); + Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1", "default"); controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); // Allow systemtest to succeed by notifying completion of system test @@ -1263,7 +1252,7 @@ public class ApplicationApiTest extends ControllerContainerTest { createAthenzDomainWithAdmin(ATHENZ_TENANT_DOMAIN, USER_ID); - Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1"); + Application application = controllerTester.createApplication(ATHENZ_TENANT_DOMAIN.getName(), "tenant1", "application1", "default"); controllerTester.authorize(ATHENZ_TENANT_DOMAIN, screwdriverId, ApplicationAction.deploy, application); // Allow systemtest to succeed by notifying completion of system test @@ -1503,14 +1492,14 @@ public class ApplicationApiTest extends ControllerContainerTest { .data("{\"athensDomain\":\"domain1\", \"property\":\"property1\"}") .oktaAccessToken(OKTA_AT), new File("tenant-without-applications.json")); - tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", POST) + tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", POST) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), new File("application-reference.json")); addScrewdriverUserToDeployRole(SCREWDRIVER_ID, ATHENZ_TENANT_DOMAIN, new com.yahoo.vespa.hosted.controller.api.identifiers.ApplicationId("application1")); - return ApplicationId.from("tenant1", "application1", "default"); + return ApplicationId.from("tenant1", "application1", "instance1"); } private void startAndTestChange(ContainerControllerTester controllerTester, ApplicationId application, @@ -1528,8 +1517,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // system-test - String testPath = String.format("/application/v4/tenant/%s/application/%s/environment/test/region/us-east-1/instance/default", - application.tenant().value(), application.application().value()); + String testPath = String.format("/application/v4/tenant/%s/application/%s/instance/%s/environment/test/region/us-east-1", + application.tenant().value(), application.application().value(), application.instance().value()); tester.assertResponse(request(testPath, POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), @@ -1543,8 +1532,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .submit(); // staging - String stagingPath = String.format("/application/v4/tenant/%s/application/%s/environment/staging/region/us-east-3/instance/default", - application.tenant().value(), application.application().value()); + String stagingPath = String.format("/application/v4/tenant/%s/application/%s/instance/%s/environment/staging/region/us-east-3", + application.tenant().value(), application.application().value(), application.instance().value()); tester.assertResponse(request(stagingPath, POST) .data(deployData) .screwdriverIdentity(SCREWDRIVER_ID), @@ -1573,7 +1562,9 @@ public class ApplicationApiTest extends ControllerContainerTest { List hostnames = new ArrayList<>(); hostnames.add("host1"); hostnames.add("host2"); - clusterInfo.put(ClusterSpec.Id.from("cluster1"), new ClusterInfo("flavor1", 37, 2, 4, 50, ClusterSpec.Type.content, hostnames)); + clusterInfo.put(ClusterSpec.Id.from("cluster1"), + new ClusterInfo("flavor1", 37, 2, 4, 50, + ClusterSpec.Type.content, hostnames)); Map clusterUtils = new HashMap<>(); clusterUtils.put(ClusterSpec.Id.from("cluster1"), new ClusterUtilization(0.3, 0.6, 0.4, 0.3)); DeploymentMetrics metrics = new DeploymentMetrics(1, 2, 3, 4, 5, @@ -1621,7 +1612,10 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void updateContactInformation() { - Contact contact = new Contact(URI.create("www.contacts.tld/1234"), URI.create("www.properties.tld/1234"), URI.create("www.issues.tld/1234"), List.of(List.of("alice"), List.of("bob")), "queue", Optional.empty()); + Contact contact = new Contact(URI.create("www.contacts.tld/1234"), + URI.create("www.properties.tld/1234"), + URI.create("www.issues.tld/1234"), + List.of(List.of("alice"), List.of("bob")), "queue", Optional.empty()); tester.controller().tenants().lockIfPresent(TenantName.from("tenant2"), LockedTenant.Athenz.class, lockedTenant -> tester.controller().tenants().store(lockedTenant.with(contact))); @@ -1629,7 +1623,10 @@ public class ApplicationApiTest extends ControllerContainerTest { private void registerContact(long propertyId) { PropertyId p = new PropertyId(String.valueOf(propertyId)); - contactRetriever().addContact(p, new Contact(URI.create("www.issues.tld/" + p.id()), URI.create("www.contacts.tld/" + p.id()), URI.create("www.properties.tld/" + p.id()), List.of(Collections.singletonList("alice"), + contactRetriever().addContact(p, new Contact(URI.create("www.issues.tld/" + p.id()), + URI.create("www.contacts.tld/" + p.id()), + URI.create("www.properties.tld/" + p.id()), + List.of(Collections.singletonList("alice"), Collections.singletonList("bob")), "queue", Optional.empty())); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json index eb53ff7161e..e673a610f87 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-nodes.json @@ -1,7 +1,7 @@ { "nodes": [ { - "hostname": "host-tenant1:application1:default-prod.us-central-1", + "hostname": "host-tenant1:application1:instance1-prod.us-central-1", "state": "active", "orchestration": "unorchestrated", "version": "6.1", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json index 7117cc22507..60243633614 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-reference.json @@ -1,6 +1,6 @@ { "tenant": "tenant1", "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default" + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json index b52fec761d8..383a1b667f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application-without-change-multiple-deployments.json @@ -1,8 +1,8 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -241,8 +241,8 @@ }, "environment": "prod", "region": "us-west-1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-west-1/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1" }, { "bcpStatus": { @@ -250,8 +250,8 @@ }, "environment": "prod", "region": "us-east-3", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-east-3/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3" } ], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 4b2cb397b5b..5d1819bf0f2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -1,8 +1,8 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", "source": { "gitRepository": "repository1", "gitBranch": "master", @@ -226,8 +226,8 @@ { "environment": "dev", "region": "us-west-1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1" }, { "bcpStatus": { @@ -235,8 +235,8 @@ }, "environment": "prod", "region": "us-central-1", - "instance": "default", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default" + "instance": "instance1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1" } ], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json index fa903b61825..c90d5759791 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json @@ -1,8 +1,8 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", - "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/", + "instance": "instance1", + "deployments": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/", "source": { "gitRepository": "repository1", "gitBranch": "master", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json index 2b17d55627a..b051fb38a41 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json @@ -1,4 +1,4 @@ { "error-code": "BAD_REQUEST", - "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-west-1 in default, zone prod.us-central-1 in default" + "message": "Could not delete 'application 'tenant1.application1.instance1'': It has active deployments in: zone dev.us-west-1 in default, zone prod.us-central-1 in default" } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json index ef0112f5ff8..65ecb3c6979 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted.json @@ -1,4 +1,4 @@ { - "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1", + "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1", "run": 1 } \ No newline at end of file diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index 6caac3bd532..25948e998f1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -1,7 +1,7 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", + "instance": "instance1", "environment": "prod", "region": "us-central-1", "endpoints": [], @@ -12,8 +12,8 @@ "http://global-endpoint.vespa.yahooapis.com:4080", "http://alias-endpoint.vespa.yahooapis.com:4080" ], - "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json index 67c71ee3880..1a2025e4de2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json @@ -1,7 +1,7 @@ { "tenant": "tenant1", "application": "application1", - "instance": "default", + "instance": "instance1", "environment": "dev", "region": "us-west-1", "endpoints": [], @@ -12,8 +12,8 @@ "http://global-endpoint.vespa.yahooapis.com:4080", "http://alias-endpoint.vespa.yahooapis.com:4080" ], - "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-west-1&application=tenant1.application1", + "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=dev®ion=us-west-1&application=tenant1.application1.instance1", "version": "(ignore)", "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json index 048ddcbc5c5..2df97a6c765 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-delete.json @@ -1 +1 @@ -{"message":"Successfully set tenant1.application1 in prod.us-west-1 in service"} +{"message":"Successfully set tenant1.application1.instance1 in prod.us-west-1 in service"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json index f67b8dd56d9..6a41b0000e4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-put.json @@ -1 +1 @@ -{"message":"Successfully set tenant1.application1 in prod.us-west-1 out of service"} +{"message":"Successfully set tenant1.application1.instance1 in prod.us-west-1 out of service"} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json index a5e4dd23079..c5cfd5981c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/jobs.json @@ -65,10 +65,10 @@ "tasks": { "deploy": "running" }, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test/run/1" + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1" } ], - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test" }, "staging-test": { "runs": [ @@ -101,10 +101,10 @@ "report": "unfinished" }, "tasks": {}, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test/run/1" + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test/run/1" } ], - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/staging-test" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/staging-test" }, "us-west-1": { "runs": [ @@ -126,7 +126,7 @@ } } ], - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/production-us-west-1" + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/production-us-west-1" } }, "devJobs": {} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 9b08fccf883..4810c8f92b2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -4,7 +4,7 @@ }, "tenant": "tenant1", "application": "application1", - "instance": "default", + "instance": "instance1", "environment": "prod", "region": "us-central-1", "endpoints": [], @@ -15,8 +15,8 @@ "http://global-endpoint.vespa.yahooapis.com:4080", "http://alias-endpoint.vespa.yahooapis.com:4080" ], - "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default", - "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1", + "nodes": "http://localhost:8080/zone/v2/prod/us-central-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.instance1", + "yamasUrl": "http://monitoring-system.test/?environment=prod®ion=us-central-1&application=tenant1.application1.instance1", "version": "(ignore)", "revision": "(ignore)", "deployTimeEpochMs": "(ignore)", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json index 782d8e42aa3..1a434afafbb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/services.json @@ -3,10 +3,10 @@ { "name": "cluster1", "type": "content", - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/container-clustercontroller-6s8slgtps7ry8uh6lx21ejjiv/cluster/v2/cluster1", "services": [ { - "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", + "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/service/storagenode-awe3slno6mmq2fye191y324jl/state/v1/", "serviceType": "storagenode", "serviceName": "storagenode", "configId": "cluster1/storage/0", diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json index 107d969e8ad..800c3976188 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/system-test-job.json @@ -28,6 +28,6 @@ "tasks": { "deploy": "running" }, - "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default/job/system-test/run/1" + "log": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1/job/system-test/run/1" } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json index 0a99248b012..edd3d7cc34f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/tenant-with-application.json @@ -7,8 +7,8 @@ { "tenant": "tenant1", "application":"application1", - "instance":"default", - "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/default" + "instance":"instance1", + "url":"http://localhost:8080/application/v4/tenant/tenant1/application/application1/instance/instance1" } ] } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java index 6eb7663e4ab..35ec5b0e37e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/BadgeApiTest.java @@ -20,7 +20,7 @@ public class BadgeApiTest extends ControllerContainerTest { @Test public void testBadgeApi() { ContainerControllerTester tester = new ContainerControllerTester(container, responseFiles); - Application application = tester.createApplication("domain", "tenant", "application"); + Application application = tester.createApplication("domain", "tenant", "application", "default"); ApplicationPackage packageWithService = new ApplicationPackageBuilder() .environment(Environment.prod) .athenzIdentity(com.yahoo.config.provision.AthenzDomain.from("domain"), AthenzService.from("service")) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index 73977d7c2fa..fa3848a6ba5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -39,12 +39,9 @@ public class DeploymentApiTest extends ControllerContainerTest { .build(); // 3 applications deploy on current system version - Application failingApplication = tester.createApplication("domain1", "tenant1", - "application1"); - Application productionApplication = tester.createApplication("domain2", "tenant2", - "application2"); - Application applicationWithoutDeployment = tester.createApplication("domain3", "tenant3", - "application3"); + Application failingApplication = tester.createApplication("domain1", "tenant1", "application1", "default"); + Application productionApplication = tester.createApplication("domain2", "tenant2", "application2", "default"); + Application applicationWithoutDeployment = tester.createApplication("domain3", "tenant3", "application3", "default"); tester.deployCompletely(failingApplication, applicationPackage, 1L, false); tester.deployCompletely(productionApplication, applicationPackage, 2L, false); -- cgit v1.2.3 From 7ee992b7ac5554c079b446769364b1f0c325d814 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 14 Jun 2019 12:56:48 +0200 Subject: Avoid using classpath to read test data --- .../vespa/hosted/controller/deployment/TestConfigSerializerTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java index bc411d4377d..d4550ecc338 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java @@ -9,6 +9,8 @@ import org.junit.Test; import java.io.IOException; import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.List; import java.util.Map; @@ -28,7 +30,7 @@ public class TestConfigSerializerTest { Map.of(zone, Map.of(ClusterSpec.Id.from("ai"), URI.create("https://server/"))), Map.of(zone, List.of("facts"))); - byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes(); + byte[] expected = Files.readAllBytes(Paths.get("src/test/resources/testConfig.json")); assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))), new String(json)); } -- cgit v1.2.3 From c186d048a0eebe5c6f78bc5407af6229b092c68c Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Thu, 13 Jun 2019 17:18:47 +0200 Subject: Billing handler and maintainer --- .../integration/organization/BillingHandler.java | 12 +++++++ .../organization/MockBillingHandler.java | 13 ++++++++ .../controller/maintenance/BillingMaintainer.java | 39 ++++++++++++++++++++++ .../maintenance/ControllerMaintenance.java | 6 +++- .../restapi/ControllerContainerTest.java | 1 + .../restapi/controller/responses/maintenance.json | 3 ++ 6 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java new file mode 100644 index 00000000000..7141b48157b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java @@ -0,0 +1,12 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author olaa + */ +public interface BillingHandler { + + void handleBilling(ApplicationId applicationId, String customerId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java new file mode 100644 index 00000000000..d4c54deac44 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author olaa + */ +public class MockBillingHandler implements BillingHandler { + + @Override + public void handleBilling(ApplicationId applicationId, String customerId) {} +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java new file mode 100644 index 00000000000..8b902554a7f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java @@ -0,0 +1,39 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.SystemName; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingHandler; +import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; + +import java.time.Duration; +import java.util.EnumSet; + +/** + * @author olaa + */ +public class BillingMaintainer extends Maintainer { + + private final BillingHandler billingHandler; + + public BillingMaintainer(Controller controller, Duration interval, JobControl jobControl, BillingHandler billingHandler) { + super(controller, interval, jobControl, BillingMaintainer.class.getSimpleName(), EnumSet.of(SystemName.cd)); + this.billingHandler = billingHandler; + } + + @Override + public void maintain() { + controller().tenants().asList() + .stream() + .filter(tenant -> tenant instanceof CloudTenant) + .map(tenant -> (CloudTenant) tenant) + .forEach(cloudTenant -> controller().applications().asList(cloudTenant.name()) + .stream() + .forEach( application -> { + billingHandler.handleBilling(application.id(), cloudTenant.billingInfo().customerId()); + }) + ); + } +} + + diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index c1f896e6593..e4d396e7e8c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,6 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingHandler; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; @@ -13,7 +14,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -55,6 +55,7 @@ public class ControllerMaintenance extends AbstractComponent { private final CostReportMaintainer costReportMaintainer; private final ResourceMeterMaintainer resourceMeterMaintainer; private final NameServiceDispatcher nameServiceDispatcher; + private final BillingMaintainer billingMaintainer; @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, @@ -64,6 +65,7 @@ public class ControllerMaintenance extends AbstractComponent { ContactRetriever contactRetriever, CostReportConsumer reportConsumer, ResourceSnapshotConsumer resourceSnapshotConsumer, + BillingHandler billingHandler, SelfHostedCostConfig selfHostedCostConfig) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; @@ -86,6 +88,7 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); + billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(15), jobControl, billingHandler); } public Upgrader upgrader() { return upgrader; } @@ -114,6 +117,7 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer.deconstruct(); resourceMeterMaintainer.deconstruct(); nameServiceDispatcher.deconstruct(); + billingMaintainer.deconstruct(); } /** Create one OS upgrader per cloud found in the zone registry of controller */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index e974f55c9f6..cd919d78e86 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -72,6 +72,7 @@ public class ControllerContainerTest { " \n" + " \n" + " \n" + + " \n" + " \n" + " \n" + " \n" + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 01b063c84e1..d4f3e20ac14 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -3,6 +3,9 @@ { "name": "ApplicationOwnershipConfirmer" }, + { + "name": "BillingMaintainer" + }, { "name": "ClusterInfoMaintainer" }, -- cgit v1.2.3 From 88ef55cb45e9998df62bf8f76b0244861b6f4e9a Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Fri, 14 Jun 2019 13:00:48 +0200 Subject: Expect exceptions from RoutingGenerator when looking for endpoints --- .../vespa/hosted/controller/ApplicationController.java | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 9edff876fb4..7e2b04c5a79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -548,19 +548,19 @@ public class ApplicationController { .orElse(id.applicationId().instance().isTester())) throw new NotExistsException("Deployment", id.toString()); + // TODO jvenstad: Swap to use routingPolicies first, when this is ready. try { - return Optional.of(routingGenerator.clusterEndpoints(id)) - .filter(endpoints -> ! endpoints.isEmpty()) - .orElseGet(() -> routingPolicies.get(id).stream() - .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), - policy -> policy.endpointIn(controller.system()).url()))); + var endpoints = routingGenerator.clusterEndpoints(id); + if ( ! endpoints.isEmpty()) + return endpoints; } catch (RuntimeException e) { - log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " - + Exceptions.toMessageString(e)); - return Collections.emptyMap(); + log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": " + Exceptions.toMessageString(e)); } + return routingPolicies.get(id).stream() + .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) + .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + policy -> policy.endpointIn(controller.system()).url())); } /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */ -- cgit v1.2.3 From ec99c240f12313e86f39790af8ca3d5463cd2173 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 14 Jun 2019 14:44:01 +0200 Subject: Simplify --- .../src/main/java/com/yahoo/config/provision/SystemName.java | 3 ++- .../hosted/controller/maintenance/ContactInformationMaintainer.java | 4 ++-- .../vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java index 0f6a87020da..29e48ae0316 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/SystemName.java @@ -72,6 +72,7 @@ public enum SystemName { public static Set all() { return EnumSet.allOf(SystemName.class); } public static Set allOf(Predicate predicate) { - return Stream.of(values()).filter(predicate::test).collect(Collectors.toSet()); + return Stream.of(values()).filter(predicate).collect(Collectors.toUnmodifiableSet()); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java index 7dfbc135df9..0080d7c23c2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ContactInformationMaintainer.java @@ -11,9 +11,9 @@ import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.yolean.Exceptions; import java.time.Duration; -import java.util.EnumSet; import java.util.Objects; import java.util.Optional; +import java.util.function.Predicate; import java.util.logging.Logger; /** @@ -28,7 +28,7 @@ public class ContactInformationMaintainer extends Maintainer { private final ContactRetriever contactRetriever; public ContactInformationMaintainer(Controller controller, Duration interval, JobControl jobControl, ContactRetriever contactRetriever) { - super(controller, interval, jobControl, null, EnumSet.of(SystemName.cd, SystemName.main)); + super(controller, interval, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.contactRetriever = Objects.requireNonNull(contactRetriever, "organization must be non-null"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index a0520f324be..9302ecbe738 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -18,6 +18,7 @@ import java.time.Clock; import java.time.Duration; import java.time.Instant; import java.util.*; +import java.util.function.Predicate; import java.util.stream.Collectors; import static com.yahoo.yolean.Exceptions.uncheck; @@ -46,7 +47,7 @@ public class ResourceMeterMaintainer extends Maintainer { Clock clock, Metric metric, ResourceSnapshotConsumer resourceSnapshotConsumer) { - super(controller, interval, jobControl, ResourceMeterMaintainer.class.getSimpleName(), Set.of(SystemName.cd, SystemName.main)); + super(controller, interval, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.clock = clock; this.nodeRepository = nodeRepository; this.metric = metric; -- cgit v1.2.3 From a7a43d6d2aa56ffdc4881357e93c502aab0a6026 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 14 Jun 2019 14:44:11 +0200 Subject: Disable ClusterUtilizationMaintainer in public --- .../hosted/controller/maintenance/ClusterUtilizationMaintainer.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java index 7955505a2b0..7cf710623ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterUtilizationMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; @@ -13,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import java.time.Duration; import java.util.HashMap; import java.util.Map; +import java.util.function.Predicate; /** * Fetch utilization metrics and update applications with this data. @@ -24,7 +26,7 @@ public class ClusterUtilizationMaintainer extends Maintainer { private final Controller controller; public ClusterUtilizationMaintainer(Controller controller, Duration duration, JobControl jobControl) { - super(controller, duration, jobControl); + super(controller, duration, jobControl, null, SystemName.allOf(Predicate.not(SystemName::isPublic))); this.controller = controller; } -- cgit v1.2.3 From c1d8ed07cb0ca74573a4740778f642586eed2595 Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Fri, 14 Jun 2019 15:01:53 +0200 Subject: Changed maintenance interval --- .../vespa/hosted/controller/maintenance/ControllerMaintenance.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index e4d396e7e8c..609cd832e60 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -88,7 +88,7 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); - billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(15), jobControl, billingHandler); + billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billingHandler); } public Upgrader upgrader() { return upgrader; } -- cgit v1.2.3 From 7a865b6daebb12083a7b1ac7e6cc544e9c37daf5 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 14 Jun 2019 16:43:48 +0200 Subject: Always upgrade tenant hosts --- .../hosted/controller/maintenance/InfrastructureUpgrader.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java index 159eb234aa7..b8bb9a7ef79 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/InfrastructureUpgrader.java @@ -3,11 +3,10 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.UpgradePolicy; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.config.provision.zone.UpgradePolicy; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.yolean.Exceptions; @@ -68,7 +67,9 @@ public abstract class InfrastructureUpgrader extends Maintainer { for (SystemApplication application : applications) { if (convergedOn(target, application.dependencies(), zone)) { boolean currentAppConverged = convergedOn(target, application, zone); - if (!currentAppConverged) { + // In dynamically provisioned zones there may be no tenant hosts at the time of upgrade, so we + // should always set the target version. + if (application == SystemApplication.tenantHost || !currentAppConverged) { upgrade(target, application, zone); } converged &= currentAppConverged; -- cgit v1.2.3 From 0de2f80d8931e59b88c0add703a71bd204e0a619 Mon Sep 17 00:00:00 2001 From: Jon Bratseth Date: Sat, 15 Jun 2019 19:20:52 +0200 Subject: Add a comment warning about serialization format changes --- .../src/main/java/com/yahoo/config/provision/AllocatedHosts.java | 7 +++++++ .../vespa/config/server/tenant/ContainerEndpointSerializer.java | 7 +++++++ .../hosted/controller/persistence/ApplicationSerializer.java | 7 +++++++ .../vespa/hosted/controller/persistence/AuditLogSerializer.java | 7 +++++++ .../controller/persistence/ConfidenceOverrideSerializer.java | 7 +++++++ .../yahoo/vespa/hosted/controller/persistence/LogSerializer.java | 7 +++++++ .../hosted/controller/persistence/NameServiceQueueSerializer.java | 7 +++++++ .../vespa/hosted/controller/persistence/OsVersionSerializer.java | 7 +++++++ .../hosted/controller/persistence/OsVersionStatusSerializer.java | 7 +++++++ .../hosted/controller/persistence/RoutingPolicySerializer.java | 7 +++++++ .../yahoo/vespa/hosted/controller/persistence/RunSerializer.java | 7 +++++++ .../vespa/hosted/controller/persistence/TenantSerializer.java | 7 +++++++ .../vespa/hosted/controller/persistence/VersionSerializer.java | 7 +++++++ .../hosted/controller/persistence/VersionStatusSerializer.java | 7 +++++++ .../yahoo/vespa/hosted/provision/persistence/NodeSerializer.java | 8 ++++---- 15 files changed, 102 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java index de4f3a555bd..96942c53a12 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/AllocatedHosts.java @@ -26,6 +26,13 @@ import java.util.Set; */ public class AllocatedHosts { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String mappingKey = "mapping"; private static final String hostSpecKey = "hostSpec"; private static final String hostSpecHostNameKey = "hostName"; diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java index 379af7f71ea..91f9e3c8eed 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/ContainerEndpointSerializer.java @@ -18,6 +18,13 @@ import java.util.List; */ public class ContainerEndpointSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String clusterIdField = "clusterId"; private static final String namesField = "names"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 47c7b54264e..1f20bdf5533 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -54,6 +54,13 @@ import java.util.TreeMap; */ public class ApplicationSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // Application fields private final String idField = "id"; private final String createdAtField = "createdAt"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java index 7fee9a7f9b4..d18e561ce5d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/AuditLogSerializer.java @@ -19,6 +19,13 @@ import java.util.function.Function; */ public class AuditLogSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String entriesField = "entries"; private static final String atField = "at"; private static final String principalField = "principal"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java index a87875da104..2cb981aac03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ConfidenceOverrideSerializer.java @@ -18,6 +18,13 @@ import java.util.Map; */ public class ConfidenceOverrideSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private final static String overridesField = "overrides"; public Slime toSlime(Map overrides) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java index 40781ac6e92..418038d4f1e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/LogSerializer.java @@ -27,6 +27,13 @@ import java.util.stream.Collectors; */ class LogSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String idField = "id"; private static final String typeField = "type"; private static final String timestampField = "at"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java index 3dfb5ffe5f8..e3dedd65e68 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/NameServiceQueueSerializer.java @@ -24,6 +24,13 @@ import java.util.ArrayList; */ public class NameServiceQueueSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String requestsField = "requests"; private static final String requestType = "requestType"; private static final String recordsField = "records"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java index 21f8b1bcb80..d68e24a27ea 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionSerializer.java @@ -20,6 +20,13 @@ import java.util.TreeSet; */ public class OsVersionSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionsField = "versions"; private static final String versionField = "version"; private static final String cloudField = "cloud"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java index 3e3c0df1673..88805f54d65 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/OsVersionStatusSerializer.java @@ -26,6 +26,13 @@ import java.util.TreeMap; */ public class OsVersionStatusSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionsField = "versions"; private static final String versionField = "version"; private static final String nodesField = "nodes"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index a9c6c54a44a..9cfce8dc16a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -24,6 +24,13 @@ import java.util.function.Function; */ public class RoutingPolicySerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String routingPoliciesField = "routingPolicies"; private static final String clusterField = "cluster"; private static final String canonicalNameField = "canonicalName"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java index f29af1055d0..1c95c9766f5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RunSerializer.java @@ -56,6 +56,13 @@ import static java.util.Comparator.comparing; */ class RunSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String stepsField = "steps"; private static final String applicationField = "id"; private static final String jobTypeField = "type"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java index 56e80068908..3a4e6c3954c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/TenantSerializer.java @@ -29,6 +29,13 @@ import java.util.Optional; */ public class TenantSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String nameField = "name"; private static final String typeField = "type"; private static final String athenzDomainField = "athenzDomain"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java index 5edae803fdb..e5897963254 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionSerializer.java @@ -13,6 +13,13 @@ import com.yahoo.slime.Slime; */ public class VersionSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + private static final String versionField = "version"; public Slime toSlime(Version version) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 72d38bbee5f..405a2e452d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -27,6 +27,13 @@ import java.util.Set; */ public class VersionStatusSerializer { + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + // VersionStatus fields private static final String versionsField = "versions"; diff --git a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java index d38a6e5031c..424889caf72 100644 --- a/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java +++ b/node-repository/src/main/java/com/yahoo/vespa/hosted/provision/persistence/NodeSerializer.java @@ -44,12 +44,12 @@ import java.util.function.UnaryOperator; */ public class NodeSerializer { - // WARNING: Since there are multiple config servers in a cluster and they upgrade one by one - // (and rewrite all nodes on startup), - // changes to the serialized format must be made such that what is serialized on version N+1 - // can be read by version N: + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: // - ADDING FIELDS: Always ok // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. /** The configured node flavors */ private final NodeFlavors flavors; -- cgit v1.2.3 From 0c1d41ed78ff6c1135d823b3b24b6eb8f63a5a5b Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Sun, 16 Jun 2019 16:30:15 +0200 Subject: Remove system from ZoneId --- .../com/yahoo/config/provision/zone/ZoneApi.java | 4 --- .../com/yahoo/config/provision/zone/ZoneId.java | 32 +++------------------- .../com/yahoo/config/provision/ZoneIdTest.java | 17 ------------ .../api/integration/deployment/JobType.java | 2 +- .../controller/maintenance/SystemUpgrader.java | 4 +-- .../controller/integration/ZoneFilterMock.java | 2 +- .../controller/maintenance/OsUpgraderTest.java | 26 +++++++++--------- .../controller/maintenance/SystemUpgraderTest.java | 10 +++---- .../hosted/controller/restapi/os/OsApiTest.java | 4 +-- 9 files changed, 28 insertions(+), 73 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java index 9f6ba29d8de..fd76dc10bdb 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneApi.java @@ -17,8 +17,4 @@ public interface ZoneApi { default RegionName getRegionName() { return getId().region(); } CloudName getCloudName(); - - default ZoneId toDeprecatedId() { - return ZoneId.from(getEnvironment(), getRegionName(), getCloudName(), getSystemName()); - } } diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java index 5e664e00b4c..b0ac59718aa 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneId.java @@ -20,30 +20,16 @@ public class ZoneId { private final Environment environment; private final RegionName region; - private final SystemName system; - private ZoneId(Environment environment, RegionName region, CloudName cloud, SystemName system) { + private ZoneId(Environment environment, RegionName region) { this.environment = Objects.requireNonNull(environment, "environment must be non-null"); this.region = Objects.requireNonNull(region, "region must be non-null"); - this.system = Objects.requireNonNull(system, "system must be non-null"); - } - - private ZoneId(Environment environment, RegionName region) { - this(environment, region, CloudName.defaultName(), SystemName.defaultSystem()); } public static ZoneId from(Environment environment, RegionName region) { return new ZoneId(environment, region); } - public static ZoneId from(SystemName system, Environment environment, RegionName region) { - return new ZoneId(environment, region, CloudName.defaultName(), system); - } - - public static ZoneId from(Environment environment, RegionName region, CloudName cloud, SystemName system) { - return new ZoneId(environment, region, cloud, system); - } - public static ZoneId from(String environment, String region) { return from(Environment.from(environment), RegionName.from(region)); } @@ -55,20 +41,14 @@ public class ZoneId { case 2: return from(parts[0], parts[1]); case 4: - return from(parts[2], parts[3], parts[0], parts[1]); + // Deprecated: parts[0] == cloud, parts[1] == system + // TODO: Figure out whether this can be removed + return from(parts[2], parts[3]); default: throw new IllegalArgumentException("Cannot deserialize zone id '" + value + "'"); } } - public static ZoneId from(Environment environment, RegionName region, CloudName cloud) { - return new ZoneId(environment, region, cloud, SystemName.defaultSystem()); - } - - public static ZoneId from(String environment, String region, String cloud, String system) { - return new ZoneId(Environment.from(environment), RegionName.from(region), CloudName.from(cloud), SystemName.from(system)); - } - public static ZoneId defaultId() { return new ZoneId(Environment.defaultEnvironment(), RegionName.defaultName()); } @@ -81,10 +61,6 @@ public class ZoneId { return region; } - public SystemName system() { - return system; - } - /** Returns the serialised value of this. Inverse of {@code ZoneId.from(String value)}. */ public String value() { return environment + "." + region; diff --git a/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java b/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java index 27d45ba7d7d..434badbe9bf 100644 --- a/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java +++ b/config-provisioning/src/test/java/com/yahoo/config/provision/ZoneIdTest.java @@ -26,17 +26,6 @@ public class ZoneIdTest { ZoneId zoneId = ZoneId.from(environment, region); assertEquals(region, zoneId.region()); assertEquals(environment, zoneId.environment()); - assertEquals(SystemName.defaultSystem(), zoneId.system()); - - ZoneId zoneIdWithSystem = ZoneId.from(system, environment, region); - assertEquals(region, zoneIdWithSystem.region()); - assertEquals(environment, zoneIdWithSystem.environment()); - assertEquals(system, zoneIdWithSystem.system()); - - ZoneId zoneIdWithCloudAndSystem = ZoneId.from(environment, region, cloud, system); - assertEquals(region, zoneIdWithCloudAndSystem.region()); - assertEquals(environment, zoneIdWithCloudAndSystem.environment()); - assertEquals(system, zoneIdWithCloudAndSystem.system()); } @Test @@ -45,12 +34,6 @@ public class ZoneIdTest { assertEquals(environment.value() + "." + region.value(), zoneId.value()); assertEquals(ZoneId.from(zoneId.value()), zoneId); - ZoneId zoneIdWithCloudAndSystem = ZoneId.from(environment, region, cloud, system); - assertEquals(environment.value() + "." + region.value(), zoneIdWithCloudAndSystem.value()); - assertEquals(ZoneId.from(zoneIdWithCloudAndSystem.value()), zoneIdWithCloudAndSystem); - // TODO: Expect cloud and system to be part of deserialized value when the new format is supported everywhere - //assertEquals(cloud.value() + "." + system.name() + "." + environment.value() + "." + region.value() , zoneId.value()); - String serializedZoneId = "some.illegal.value"; expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("Cannot deserialize zone id '" + serializedZoneId + "'"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java index 94e111455ac..a1beef23dbb 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/deployment/JobType.java @@ -154,7 +154,7 @@ public enum JobType { case test: return Optional.of(systemTest); case staging: return Optional.of(stagingTest); } - return from(system, ZoneId.from(system, environment, region)); + return from(system, ZoneId.from(environment, region)); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java index 5a40dd591fd..156e8d0d242 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgrader.java @@ -34,7 +34,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { if (minVersion(zone, application, Node::wantedVersion).map(target::isAfter) .orElse(true)) { log.info(String.format("Deploying %s version %s in %s", application.id(), target, zone.getId())); - controller().applications().deploy(application, zone.toDeprecatedId(), target); + controller().applications().deploy(application, zone.getId(), target); } } @@ -45,7 +45,7 @@ public class SystemUpgrader extends InfrastructureUpgrader { if (minVersion.isEmpty()) return true; return minVersion.get().equals(target) - && application.configConvergedIn(zone.toDeprecatedId(), controller(), Optional.of(target)); + && application.configConvergedIn(zone.getId(), controller(), Optional.of(target)); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 700e6e9cb42..57f29fb72af 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -83,7 +83,7 @@ public class ZoneFilterMock implements ZoneList { @Override public List ids() { - return List.copyOf(zones.stream().map(ZoneApi::toDeprecatedId).collect(Collectors.toList())); + return List.copyOf(zones.stream().map(ZoneApi::getId).collect(Collectors.toList())); } @Override diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java index 39ff29f4ae0..38aa4af4756 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsUpgraderTest.java @@ -57,16 +57,16 @@ public class OsUpgraderTest { ); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId(), zone5.toDeprecatedId()), + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.tenantHost)); // Add system applications that exist in a real system, but are currently not upgraded - tester.configServer().addNodes(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId(), zone5.toDeprecatedId()), + tester.configServer().addNodes(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId(), zone5.getId()), List.of(SystemApplication.configServer)); // Fail a few nodes. Failed nodes should not affect versions - failNodeIn(zone1.toDeprecatedId(), SystemApplication.tenantHost); - failNodeIn(zone3.toDeprecatedId(), SystemApplication.tenantHost); + failNodeIn(zone1.getId(), SystemApplication.tenantHost); + failNodeIn(zone3.getId(), SystemApplication.tenantHost); // New OS version released Version version1 = Version.fromString("7.1"); @@ -78,37 +78,37 @@ public class OsUpgraderTest { // zone 1: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone1.toDeprecatedId()); + assertWanted(version1, SystemApplication.tenantHost, zone1.getId()); // Other zones remain on previous version (none) - assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()); + assertWanted(Version.emptyVersion, SystemApplication.proxy, zone2.getId(), zone3.getId(), zone4.getId()); // zone 1: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone1.toDeprecatedId()); + completeUpgrade(version1, SystemApplication.tenantHost, zone1.getId()); statusUpdater.maintain(); assertEquals(2, nodesOn(version1).size()); assertEquals(11, nodesOn(Version.emptyVersion).size()); // zone 2 and 3: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.proxy, zone2.toDeprecatedId(), zone3.toDeprecatedId()); + assertWanted(version1, SystemApplication.proxy, zone2.getId(), zone3.getId()); // zone 4: still on previous version - assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4.toDeprecatedId()); + assertWanted(Version.emptyVersion, SystemApplication.tenantHost, zone4.getId()); // zone 2 and 3: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone2.toDeprecatedId(), zone3.toDeprecatedId()); + completeUpgrade(version1, SystemApplication.tenantHost, zone2.getId(), zone3.getId()); // zone 4: begins upgrading osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone4.toDeprecatedId()); + assertWanted(version1, SystemApplication.tenantHost, zone4.getId()); // zone 4: completes upgrade - completeUpgrade(version1, SystemApplication.tenantHost, zone4.toDeprecatedId()); + completeUpgrade(version1, SystemApplication.tenantHost, zone4.getId()); // Next run does nothing as all zones are upgraded osUpgrader.maintain(); - assertWanted(version1, SystemApplication.tenantHost, zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()); + assertWanted(version1, SystemApplication.tenantHost, zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()); statusUpdater.maintain(); assertTrue("All nodes on target version", tester.controller().osVersionStatus().nodesIn(cloud).stream() .allMatch(node -> node.version().equals(version1))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java index cb5e7cc90a1..7b817c175b8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/SystemUpgraderTest.java @@ -50,7 +50,7 @@ public class SystemUpgraderTest { Version version1 = Version.fromString("6.5"); // Bootstrap a system without host applications - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), SystemApplication.configServer, SystemApplication.proxy); // Fail a few nodes. Failed nodes should not affect versions failNodeIn(zone1, SystemApplication.configServer); @@ -144,7 +144,7 @@ public class SystemUpgraderTest { SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); // Bootstrap system - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), SystemApplication.configServer, + tester.configServer().bootstrap(List.of(zone1.getId()), SystemApplication.configServer, SystemApplication.proxy); Version version1 = Version.fromString("6.5"); tester.upgradeSystem(version1); @@ -184,7 +184,7 @@ public class SystemUpgraderTest { ); Version version1 = Version.fromString("6.5"); - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId(), zone2.toDeprecatedId(), zone3.toDeprecatedId(), zone4.toDeprecatedId()), SystemApplication.all()); + tester.configServer().bootstrap(List.of(zone1.getId(), zone2.getId(), zone3.getId(), zone4.getId()), SystemApplication.all()); tester.upgradeSystem(version1); systemUpgrader.maintain(); assertCurrentVersion(SystemApplication.all(), version1, zone1, zone2, zone3, zone4); @@ -282,7 +282,7 @@ public class SystemUpgraderTest { public void does_not_deploy_proxy_app_in_zones_without_proxy() { List applications = List.of( SystemApplication.configServerHost, SystemApplication.configServer, SystemApplication.tenantHost); - tester.configServer().bootstrap(List.of(zone1.toDeprecatedId()), applications); + tester.configServer().bootstrap(List.of(zone1.getId()), applications); tester.configServer().disallowConvergenceCheck(SystemApplication.proxy.id()); SystemUpgrader systemUpgrader = systemUpgrader(UpgradePolicy.create().upgrade(zone1)); @@ -309,7 +309,7 @@ public class SystemUpgraderTest { private void convergeServices(SystemApplication application, ZoneApi... zones) { for (ZoneApi zone : zones) { - tester.controllerTester().configServer().convergeServices(application.id(), zone.toDeprecatedId()); + tester.controllerTester().configServer().convergeServices(application.id(), zone.getId()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java index b2dfd7b4cb6..745a7af203b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiTest.java @@ -81,12 +81,12 @@ public class OsApiTest extends ControllerContainerTest { // Status is updated after some zones are upgraded upgradeAndUpdateStatus(); - completeUpgrade(zone1.toDeprecatedId()); + completeUpgrade(zone1.getId()); assertFile(new Request("http://localhost:8080/os/v1/"), "versions-partially-upgraded.json"); // All zones are upgraded upgradeAndUpdateStatus(); - completeUpgrade(zone2.toDeprecatedId(), zone3.toDeprecatedId()); + completeUpgrade(zone2.getId(), zone3.getId()); assertFile(new Request("http://localhost:8080/os/v1/"), "versions-all-upgraded.json"); // Downgrade with force is permitted -- cgit v1.2.3 From c91aa7f73f6e3fc4f7771fe51c9167e25673367d Mon Sep 17 00:00:00 2001 From: Ola Aunrønning Date: Tue, 18 Jun 2019 14:34:42 +0200 Subject: Rename BillingHandler --- .../controller/api/integration/organization/Billing.java | 12 ++++++++++++ .../api/integration/organization/BillingHandler.java | 12 ------------ .../api/integration/organization/MockBilling.java | 13 +++++++++++++ .../api/integration/organization/MockBillingHandler.java | 13 ------------- .../hosted/controller/maintenance/BillingMaintainer.java | 10 +++++----- .../controller/maintenance/ControllerMaintenance.java | 6 +++--- .../hosted/controller/restapi/ControllerContainerTest.java | 2 +- 7 files changed, 34 insertions(+), 34 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java new file mode 100644 index 00000000000..f716458542c --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/Billing.java @@ -0,0 +1,12 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author olaa + */ +public interface Billing { + + void handleBilling(ApplicationId applicationId, String customerId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java deleted file mode 100644 index 7141b48157b..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/BillingHandler.java +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.organization; - -import com.yahoo.config.provision.ApplicationId; - -/** - * @author olaa - */ -public interface BillingHandler { - - void handleBilling(ApplicationId applicationId, String customerId); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java new file mode 100644 index 00000000000..20b77703160 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBilling.java @@ -0,0 +1,13 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.api.integration.organization; + +import com.yahoo.config.provision.ApplicationId; + +/** + * @author olaa + */ +public class MockBilling implements Billing { + + @Override + public void handleBilling(ApplicationId applicationId, String customerId) {} +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java deleted file mode 100644 index d4c54deac44..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/organization/MockBillingHandler.java +++ /dev/null @@ -1,13 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.organization; - -import com.yahoo.config.provision.ApplicationId; - -/** - * @author olaa - */ -public class MockBillingHandler implements BillingHandler { - - @Override - public void handleBilling(ApplicationId applicationId, String customerId) {} -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java index 8b902554a7f..c6956293adf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/BillingMaintainer.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingHandler; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; import com.yahoo.vespa.hosted.controller.tenant.CloudTenant; import java.time.Duration; @@ -14,11 +14,11 @@ import java.util.EnumSet; */ public class BillingMaintainer extends Maintainer { - private final BillingHandler billingHandler; + private final Billing billing; - public BillingMaintainer(Controller controller, Duration interval, JobControl jobControl, BillingHandler billingHandler) { + public BillingMaintainer(Controller controller, Duration interval, JobControl jobControl, Billing billing) { super(controller, interval, jobControl, BillingMaintainer.class.getSimpleName(), EnumSet.of(SystemName.cd)); - this.billingHandler = billingHandler; + this.billing = billing; } @Override @@ -30,7 +30,7 @@ public class BillingMaintainer extends Maintainer { .forEach(cloudTenant -> controller().applications().asList(cloudTenant.name()) .stream() .forEach( application -> { - billingHandler.handleBilling(application.id(), cloudTenant.billingInfo().customerId()); + billing.handleBilling(application.id(), cloudTenant.billingInfo().customerId()); }) ); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 609cd832e60..e840deb062c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,7 +5,7 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.BillingHandler; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; @@ -65,7 +65,7 @@ public class ControllerMaintenance extends AbstractComponent { ContactRetriever contactRetriever, CostReportConsumer reportConsumer, ResourceSnapshotConsumer resourceSnapshotConsumer, - BillingHandler billingHandler, + Billing billing, SelfHostedCostConfig selfHostedCostConfig) { Duration maintenanceInterval = Duration.ofMinutes(maintainerConfig.intervalMinutes()); this.jobControl = jobControl; @@ -88,7 +88,7 @@ public class ControllerMaintenance extends AbstractComponent { costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); - billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billingHandler); + billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing); } public Upgrader upgrader() { return upgrader; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index cd919d78e86..68f0738f7a5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -72,7 +72,7 @@ public class ControllerContainerTest { " \n" + " \n" + " \n" + - " \n" + + " \n" + " \n" + " \n" + " \n" + -- cgit v1.2.3 From a22f03a4b4d054f12118c7c1d03ac8ec4ccfea56 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 09:54:44 +0200 Subject: Test and fix parsing, and rename to MavenRepository --- .../api/integration/maven/MavenRepository.java | 12 +++++ .../controller/api/integration/maven/Metadata.java | 5 +- .../api/integration/maven/Repository.java | 12 ----- .../api/integration/stubs/MockMavenRepository.java | 23 +++++++++ .../api/integration/stubs/MockRepository.java | 23 --------- .../api/integration/maven/MetadataTest.java | 56 ++++++++++++++++++++++ .../maintenance/CostReportMaintainer.java | 3 +- 7 files changed, 95 insertions(+), 39 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Repository.java create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRepository.java create mode 100644 controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java new file mode 100644 index 00000000000..ef667bac8c5 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java @@ -0,0 +1,12 @@ +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +/** + * A Maven repository which keeps released artifacts. + * + * @author jonmv + */ +public interface MavenRepository { + + Metadata getMetadata(ArtifactId id); + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java index e837eaf875e..fd84a05db6a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Metadata.java @@ -31,12 +31,11 @@ public class Metadata { /** Creates a new Metadata object from the given XML document. */ public static Metadata fromXml(String xml) { - Element root = XML.getDocument(xml).getDocumentElement(); - Element metadata = XML.getChild(root, "metadata"); + Element metadata = XML.getDocument(xml).getDocumentElement(); ArtifactId id = new ArtifactId(XML.getValue(XML.getChild(metadata, "groupId")), XML.getValue(XML.getChild(metadata, "artifactId"))); List versions = new ArrayList<>(); - for (Element version : XML.getChildren(XML.getChild(metadata, "versioning"), "versions")) + for (Element version : XML.getChildren(XML.getChild(XML.getChild(metadata, "versioning"), "versions"))) versions.add(Version.fromString(XML.getValue(version))); return new Metadata(id, versions); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Repository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Repository.java deleted file mode 100644 index 41a13d41814..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/Repository.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.yahoo.vespa.hosted.controller.api.integration.maven; - -/** - * A Maven repository which keeps released artifacts. - * - * @author jonmv - */ -public interface Repository { - - Metadata getMetadata(ArtifactId id); - -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java new file mode 100644 index 00000000000..2a8b47af777 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java @@ -0,0 +1,23 @@ +package com.yahoo.vespa.hosted.controller.api.integration.stubs; + +import com.yahoo.component.Version; +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; + +import java.util.List; + +/** + * Mock repository for maven artifacts, that returns a static metadata. + * + * @author jonmv + */ +public class MockMavenRepository implements MavenRepository { + + @Override + public Metadata getMetadata(ArtifactId id) { + return new Metadata(id, List.of(Version.fromString("3.2.1"), + Version.fromString("1.2.3"))); + } + +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRepository.java deleted file mode 100644 index 9a93cd94aaa..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockRepository.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.yahoo.vespa.hosted.controller.api.integration.stubs; - -import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; -import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; -import com.yahoo.vespa.hosted.controller.api.integration.maven.Repository; - -import java.util.List; - -/** - * Mock repository for maven artifacts, that returns a static metadata. - * - * @author jonmv - */ -public class MockRepository implements Repository { - - @Override - public Metadata getMetadata(ArtifactId id) { - return new Metadata(id, List.of(Version.fromString("3.2.1"), - Version.fromString("1.2.3"))); - } - -} diff --git a/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java new file mode 100644 index 00000000000..17d0694538c --- /dev/null +++ b/controller-api/src/test/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MetadataTest.java @@ -0,0 +1,56 @@ +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +import com.yahoo.component.Version; +import org.junit.Assert; +import org.junit.Test; + +import java.net.URI; +import java.nio.file.Path; + +import static org.junit.Assert.assertEquals; + +public class MetadataTest { + + @Test + public void testParsing() { + Metadata metadata = Metadata.fromXml(metadataXml); + assertEquals("com.yahoo.vespa", metadata.id().groupId()); + assertEquals("tenant-base", metadata.id().artifactId()); + assertEquals(Version.fromString("6.297.80"), metadata.versions().get(0)); + assertEquals(Version.fromString("7.61.10"), metadata.versions().get(metadata.versions().size() - 1)); + } + + private static final String metadataXml = "\n" + + "\n" + + " com.yahoo.vespa\n" + + " tenant-base\n" + + " \n" + + " 7.61.10\n" + + " 7.61.10\n" + + " \n" + + " 6.297.80\n" + + " 6.300.15\n" + + " 6.301.8\n" + + " 6.303.29\n" + + " 6.304.14\n" + + " 6.305.35\n" + + " 6.328.65\n" + + " 6.329.64\n" + + " 6.330.51\n" + + " 7.3.19\n" + + " 7.18.17\n" + + " 7.20.129\n" + + " 7.21.18\n" + + " 7.22.18\n" + + " 7.38.38\n" + + " 7.39.5\n" + + " 7.40.41\n" + + " 7.41.15\n" + + " 7.57.40\n" + + " 7.60.51\n" + + " 7.61.10\n" + + " \n" + + " 20190619054245\n" + + " \n" + + "\n"; +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 2b26e93aeb8..30bca180c0f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -12,7 +12,8 @@ import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfi import java.time.Clock; import java.time.Duration; -import java.util.*; +import java.util.EnumSet; +import java.util.Objects; import java.util.logging.Logger; /** -- cgit v1.2.3 From 2245865a6fb46757ae5414205b9dbbb6fb05a0a5 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 09:55:23 +0200 Subject: Add MavenRepositoryClient for fetching metadata from a remote --- .../controller/versions/MavenRepositoryClient.java | 56 ++++++++++++++++++++++ .../configdefinitions/maven-repository.def | 8 ++++ .../versions/MavenRepositoryClientTest.java | 22 +++++++++ 3 files changed, 86 insertions(+) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java create mode 100644 controller-server/src/main/resources/configdefinitions/maven-repository.def create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java new file mode 100644 index 00000000000..35105cdcb2d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -0,0 +1,56 @@ +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; +import com.yahoo.vespa.hosted.controller.maven.repository.config.MavenRepositoryConfig; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; + +import static java.nio.charset.StandardCharsets.UTF_8; + +public class MavenRepositoryClient implements MavenRepository { + + private static final String artifactoryApi = "https://edge.artifactory.ouroath.com:4443/artifactory/vespa-maven-libs-release-local/"; + + private final HttpClient client; + private final URI apiUrl; + + public MavenRepositoryClient(MavenRepositoryConfig config) { + this.client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); + this.apiUrl = URI.create(config.apiUrl() + "/").normalize(); + } + + @Override + public Metadata getMetadata(ArtifactId id) { + try { + HttpRequest request = HttpRequest.newBuilder(withArtifactPath(apiUrl, id)).build(); + HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString(UTF_8)); + if (response.statusCode() != 200) + throw new RuntimeException("Status code '" + response.statusCode() + "' and body\n'''\n" + + response.body() + "\n'''\nfor request " + request); + + return Metadata.fromXml(response.body()); + } + catch (IOException | InterruptedException e) { + throw new RuntimeException(e); + } + } + + static URI withArtifactPath(URI baseUrl, ArtifactId id) { + List parts = new ArrayList<>(List.of(id.groupId().split("\\."))); + parts.add(id.artifactId()); + parts.add("maven-metadata.xml"); + return baseUrl.resolve(String.join("/", parts)); + } + +} diff --git a/controller-server/src/main/resources/configdefinitions/maven-repository.def b/controller-server/src/main/resources/configdefinitions/maven-repository.def new file mode 100644 index 00000000000..1072b14f7ab --- /dev/null +++ b/controller-server/src/main/resources/configdefinitions/maven-repository.def @@ -0,0 +1,8 @@ +# Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +namespace=vespa.hosted.controller.maven.repository.config + + +# URL to the Maven repository API that holds artifacts for tenants in the controller's system +# +apiUrl string default=https://repo.maven.apache.org/maven2/ + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java new file mode 100644 index 00000000000..026d174cb73 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClientTest.java @@ -0,0 +1,22 @@ +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; +import org.junit.Test; + +import java.net.URI; + +import static org.junit.Assert.assertEquals; + +/** + * @author jonmv + */ +public class MavenRepositoryClientTest { + + @Test + public void testUri() { + assertEquals(URI.create("https://domain:123/base/group/id/artifact-id/maven-metadata.xml"), + MavenRepositoryClient.withArtifactPath(URI.create("https://domain:123/base/"), + new ArtifactId("group.id", "artifact-id"))); + } + +} -- cgit v1.2.3 From 966a87542d2f3e6e4a0096c9420018cfbe8f671d Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 10:58:19 +0200 Subject: Keep a MavenRepository in the Controller --- .../com/yahoo/vespa/hosted/controller/Controller.java | 17 +++++++++++------ .../vespa/hosted/controller/versions/VersionStatus.java | 3 +++ .../yahoo/vespa/hosted/controller/ControllerTester.java | 4 +++- .../controller/restapi/ControllerContainerTest.java | 1 + 4 files changed, 18 insertions(+), 7 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 6f0ee75d098..d87d52f2c12 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationS import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHub; +import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mailer; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; @@ -83,6 +84,7 @@ public class Controller extends AbstractComponent { private final AuditLogger auditLogger; private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; + private final MavenRepository mavenRepository; /** * Creates a controller @@ -95,11 +97,13 @@ public class Controller extends AbstractComponent { RoutingGenerator routingGenerator, Chef chef, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, - BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource) { + BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, + MavenRepository mavenRepository) { this(curator, rotationsConfig, gitHub, zoneRegistry, configServer, metricsService, routingGenerator, chef, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, - buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource); + buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, + mavenRepository); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, @@ -109,7 +113,7 @@ public class Controller extends AbstractComponent { AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier hostnameSupplier, - Mailer mailer, FlagSource flagSource) { + Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -122,6 +126,7 @@ public class Controller extends AbstractComponent { this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.nameServiceForwarder = new NameServiceForwarder(curator); + this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); jobController = new JobController(this, runDataStore, Objects.requireNonNull(testerCloud)); applicationController = new ApplicationController(this, curator, accessControl, @@ -164,9 +169,9 @@ public class Controller extends AbstractComponent { public ZoneRegistry zoneRegistry() { return zoneRegistry; } - public NameServiceForwarder nameServiceForwarder() { - return nameServiceForwarder; - } + public NameServiceForwarder nameServiceForwarder() { return nameServiceForwarder; } + + public MavenRepository mavenRepository() { return mavenRepository; } public ApplicationView getApplicationView(String tenantName, String applicationName, String instanceName, String environment, String region) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 5e57e9ebe8e..54eee4296dd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -145,6 +145,9 @@ public class VersionStatus { statistics.version().toFullString(), e); } } + + + Collections.sort(versions); return new VersionStatus(versions); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 2b0ee741e7e..5748ad4f55c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueH import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; +import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; import com.yahoo.config.provision.zone.ZoneId; @@ -351,7 +352,8 @@ public final class ControllerTester { new MockRunDataStore(), () -> "test-controller", new MockMailer(), - new InMemoryFlagSource()); + new InMemoryFlagSource(), + new MockMavenRepository()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 68f0738f7a5..6f612005524 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -92,6 +92,7 @@ public class ControllerContainerTest { " \n" + " \n" + " \n" + + " \n" + " \n" + " http://*/deployment/v1/*\n" + " \n" + -- cgit v1.2.3 From e841033df8a4aac9d875d4161981980b878c8c61 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 13:18:50 +0200 Subject: Configure artifact to check for as well --- .../controller/api/integration/maven/MavenRepository.java | 3 ++- .../controller/api/integration/stubs/MockMavenRepository.java | 8 +++++--- .../hosted/controller/versions/MavenRepositoryClient.java | 10 ++++------ .../src/main/resources/configdefinitions/maven-repository.def | 7 +++++++ 4 files changed, 18 insertions(+), 10 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java index ef667bac8c5..853da9c4056 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java @@ -7,6 +7,7 @@ package com.yahoo.vespa.hosted.controller.api.integration.maven; */ public interface MavenRepository { - Metadata getMetadata(ArtifactId id); + /** Returns metadata about all releases of a specific artifact to this repository. */ + Metadata getMetadata(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java index 2a8b47af777..d4a52a0d77d 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java @@ -14,10 +14,12 @@ import java.util.List; */ public class MockMavenRepository implements MavenRepository { + public static final ArtifactId id = new ArtifactId("ai.vespa", "search"); + @Override - public Metadata getMetadata(ArtifactId id) { - return new Metadata(id, List.of(Version.fromString("3.2.1"), - Version.fromString("1.2.3"))); + public Metadata getMetadata() { + return new Metadata(id, List.of(Version.fromString("6.1"), + Version.fromString("6.2"))); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java index 35105cdcb2d..871cc4d58cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -1,8 +1,8 @@ package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; -import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; import com.yahoo.vespa.hosted.controller.api.integration.maven.MavenRepository; +import com.yahoo.vespa.hosted.controller.api.integration.maven.Metadata; import com.yahoo.vespa.hosted.controller.maven.repository.config.MavenRepositoryConfig; import java.io.IOException; @@ -10,28 +10,26 @@ import java.net.URI; import java.net.http.HttpClient; import java.net.http.HttpRequest; import java.net.http.HttpResponse; -import java.nio.file.Path; import java.time.Duration; import java.util.ArrayList; import java.util.List; -import java.util.stream.Stream; import static java.nio.charset.StandardCharsets.UTF_8; public class MavenRepositoryClient implements MavenRepository { - private static final String artifactoryApi = "https://edge.artifactory.ouroath.com:4443/artifactory/vespa-maven-libs-release-local/"; - private final HttpClient client; private final URI apiUrl; + private final ArtifactId id; public MavenRepositoryClient(MavenRepositoryConfig config) { this.client = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(5)).build(); this.apiUrl = URI.create(config.apiUrl() + "/").normalize(); + this.id = new ArtifactId(config.groupId(), config.artifactId()); } @Override - public Metadata getMetadata(ArtifactId id) { + public Metadata getMetadata() { try { HttpRequest request = HttpRequest.newBuilder(withArtifactPath(apiUrl, id)).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString(UTF_8)); diff --git a/controller-server/src/main/resources/configdefinitions/maven-repository.def b/controller-server/src/main/resources/configdefinitions/maven-repository.def index 1072b14f7ab..0fd2d410e9b 100644 --- a/controller-server/src/main/resources/configdefinitions/maven-repository.def +++ b/controller-server/src/main/resources/configdefinitions/maven-repository.def @@ -6,3 +6,10 @@ namespace=vespa.hosted.controller.maven.repository.config # apiUrl string default=https://repo.maven.apache.org/maven2/ +# Group ID of the artifact to list versions for +# +groupId string default=com.yahoo.vespa + +# Artifact ID of the artifact to list versions for +# +artifactId string default=tenant-base -- cgit v1.2.3 From 28144ea890d73be82f3c7a2c9a9ee7dbb491a061 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 13:19:23 +0200 Subject: Store whether versions have been released to customer-available repo for builds --- .../controller/persistence/VersionStatusSerializer.java | 3 +++ .../vespa/hosted/controller/versions/VersionStatus.java | 12 ++++++++---- .../yahoo/vespa/hosted/controller/versions/VespaVersion.java | 7 ++++++- .../controller/persistence/VersionStatusSerializerTest.java | 5 +++-- .../controller/restapi/deployment/DeploymentApiTest.java | 1 + .../vespa/hosted/controller/versions/VersionStatusTest.java | 7 ++++++- 6 files changed, 27 insertions(+), 8 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index 405a2e452d0..a4b339a69bc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -42,6 +42,7 @@ public class VersionStatusSerializer { private static final String committedAtField = "releasedAt"; private static final String isControllerVersionField = "isCurrentControllerVersion"; private static final String isSystemVersionField = "isCurrentSystemVersion"; + private static final String isReleasedField = "isReleased"; private static final String deploymentStatisticsField = "deploymentStatistics"; private static final String confidenceField = "confidence"; private static final String configServersField = "configServerHostnames"; @@ -73,6 +74,7 @@ public class VersionStatusSerializer { object.setLong(committedAtField, version.committedAt().toEpochMilli()); object.setBool(isControllerVersionField, version.isControllerVersion()); object.setBool(isSystemVersionField, version.isSystemVersion()); + object.setBool(isReleasedField, version.isReleased()); deploymentStatisticsToSlime(version.statistics(), object.setObject(deploymentStatisticsField)); object.setString(confidenceField, version.confidence().name()); configServersToSlime(version.systemApplicationHostnames(), object.setArray(configServersField)); @@ -105,6 +107,7 @@ public class VersionStatusSerializer { Instant.ofEpochMilli(object.field(committedAtField).asLong()), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), + object.field(isReleasedField).asBool(), configServersFromSlime(object.field(configServersField)), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 54eee4296dd..6ff14730285 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -12,6 +12,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; +import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; @@ -129,14 +130,17 @@ public class VersionStatus { Collection deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, controller.applications().asList()); List versions = new ArrayList<>(); + List releasedVersions = controller.mavenRepository().getMetadata().versions(); for (DeploymentStatistics statistics : deploymentStatistics) { if (statistics.version().isEmpty()) continue; try { + boolean isReleased = Collections.binarySearch(releasedVersions, statistics.version()) >= 0; VespaVersion vespaVersion = createVersion(statistics, statistics.version().equals(controllerVersion), statistics.version().equals(systemVersion), + isReleased, systemApplicationVersions.getList(statistics.version()), controller); versions.add(vespaVersion); @@ -146,8 +150,6 @@ public class VersionStatus { } } - - Collections.sort(versions); return new VersionStatus(versions); @@ -241,10 +243,11 @@ public class VersionStatus { } return versionMap.values(); } - + private static VespaVersion createVersion(DeploymentStatistics statistics, boolean isControllerVersion, - boolean isSystemVersion, + boolean isSystemVersion, + boolean isReleased, Collection configServerHostnames, Controller controller) { GitSha gitSha = controller.gitHub().getCommit(VESPA_REPO_OWNER, VESPA_REPO, statistics.version().toFullString()); @@ -263,6 +266,7 @@ public class VersionStatus { gitSha.sha, committedAt, isControllerVersion, isSystemVersion, + isReleased, configServerHostnames, confidence ); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index ffbf24be12a..117ce80adaa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -27,12 +27,13 @@ public class VespaVersion implements Comparable { private final Instant committedAt; private final boolean isControllerVersion; private final boolean isSystemVersion; + private final boolean isReleased; private final DeploymentStatistics statistics; private final ImmutableSet systemApplicationHostnames; private final Confidence confidence; public VespaVersion(DeploymentStatistics statistics, String releaseCommit, Instant committedAt, - boolean isControllerVersion, boolean isSystemVersion, + boolean isControllerVersion, boolean isSystemVersion, boolean isReleased, Collection systemApplicationHostnames, Confidence confidence) { this.statistics = statistics; @@ -40,6 +41,7 @@ public class VespaVersion implements Comparable { this.committedAt = committedAt; this.isControllerVersion = isControllerVersion; this.isSystemVersion = isSystemVersion; + this.isReleased = isReleased; this.systemApplicationHostnames = ImmutableSet.copyOf(systemApplicationHostnames); this.confidence = confidence; } @@ -102,6 +104,9 @@ public class VespaVersion implements Comparable { */ public boolean isSystemVersion() { return isSystemVersion; } + /** Returns whether the artifacts of this release are available in the configured maven repository. */ + public boolean isReleased() { return isReleased; } + /** Returns the hosts allocated to system applications (across all zones) which are currently of this version */ public Set systemApplicationHostnames() { return systemApplicationHostnames; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java index 5e6f9811376..a1e22b4fc64 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializerTest.java @@ -36,9 +36,9 @@ public class VersionStatusSerializerTest { ApplicationId.from("tenant2", "success2", "default")) ); vespaVersions.add(new VespaVersion(statistics, "dead", Instant.now(), false, false, - asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + true, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); vespaVersions.add(new VespaVersion(statistics, "cafe", Instant.now(), true, true, - asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); + false, asHostnames("cfg1", "cfg2", "cfg3"), VespaVersion.Confidence.normal)); VersionStatus status = new VersionStatus(vespaVersions); VersionStatusSerializer serializer = new VersionStatusSerializer(); VersionStatus deserialized = serializer.fromSlime(serializer.toSlime(status)); @@ -51,6 +51,7 @@ public class VersionStatusSerializerTest { assertEquals(a.committedAt().truncatedTo(MILLIS), b.committedAt()); assertEquals(a.isControllerVersion(), b.isControllerVersion()); assertEquals(a.isSystemVersion(), b.isSystemVersion()); + assertEquals(a.isReleased(), b.isReleased()); assertEquals(a.statistics(), b.statistics()); assertEquals(a.systemApplicationHostnames(), b.systemApplicationHostnames()); assertEquals(a.confidence(), b.confidence()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java index fa3848a6ba5..d6620733efe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiTest.java @@ -74,6 +74,7 @@ public class DeploymentApiTest extends ControllerContainerTest { version.committedAt(), version.isControllerVersion(), version.isSystemVersion(), + version.isReleased(), ImmutableSet.of("config1.test", "config2.test").stream() .map(HostName::from) .collect(Collectors.toSet()), diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index a365285b752..8e3dc24193f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -254,7 +254,7 @@ public class VersionStatusTest { assertTrue("Status for version without applications is removed", tester.controller().versionStatus().versions().stream() .noneMatch(vespaVersion -> vespaVersion.versionNumber().equals(version1))); - + // Another default application upgrades, raising confidence to high tester.completeUpgrade(default8, version2, "default"); tester.completeUpgrade(default9, version2, "default"); @@ -294,6 +294,11 @@ public class VersionStatusTest { assertEquals("6.2", versions.get(0).versionNumber().toString()); assertEquals("6.4", versions.get(1).versionNumber().toString()); assertEquals("6.5", versions.get(2).versionNumber().toString()); + + // Check release status is correct (static data in MockMavenRepository). + assertTrue(versions.get(0).isReleased()); + assertFalse(versions.get(1).isReleased()); + assertFalse(versions.get(2).isReleased()); } @Test -- cgit v1.2.3 From c987968d69f35c32bbfa65440bb0112caed96cac Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 13:41:28 +0200 Subject: Filter out non-released versions for compile version --- .../api/integration/maven/MavenRepository.java | 5 ++++- .../api/integration/stubs/MockMavenRepository.java | 9 +++++++-- .../restapi/application/ApplicationApiHandler.java | 23 +++++++++++++++++++++- .../controller/versions/MavenRepositoryClient.java | 7 ++++++- .../hosted/controller/versions/VersionStatus.java | 3 +-- .../responses/application2-with-patches.json | 2 +- .../application/responses/application2.json | 2 +- 7 files changed, 42 insertions(+), 9 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java index 853da9c4056..fb133f75654 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/MavenRepository.java @@ -8,6 +8,9 @@ package com.yahoo.vespa.hosted.controller.api.integration.maven; public interface MavenRepository { /** Returns metadata about all releases of a specific artifact to this repository. */ - Metadata getMetadata(); + Metadata metadata(); + + /** Returns the id of the artifact whose releases this tracks. */ + ArtifactId artifactId(); } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java index d4a52a0d77d..0ecb7ae13cc 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java @@ -17,9 +17,14 @@ public class MockMavenRepository implements MavenRepository { public static final ArtifactId id = new ArtifactId("ai.vespa", "search"); @Override - public Metadata getMetadata() { - return new Metadata(id, List.of(Version.fromString("6.1"), + public Metadata metadata() { + return new Metadata(id, List.of(Version.fromString("6.0"), Version.fromString("6.2"))); } + @Override + public ArtifactId artifactId() { + return id; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 3db8c447572..79684e705b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -97,6 +97,7 @@ import java.time.Instant; import java.util.Arrays; import java.util.Base64; import java.util.Collections; +import java.util.Comparator; import java.util.List; import java.util.Map; import java.util.Optional; @@ -515,7 +516,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { }); // Compile version. The version that should be used when building an application - object.setString("compileVersion", controller.applications().oldestInstalledPlatform(application.id()).toFullString()); + object.setString("compileVersion", compileVersion(application.id()).toFullString()); application.majorVersion().ifPresent(majorVersion -> object.setLong("majorVersion", majorVersion)); @@ -693,6 +694,26 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return controller.zoneRegistry().getMonitoringSystemUri(deploymentId); } + /** + * Returns a version at least as old as the oldest platform the given application is on, and which is released. + * + * If no known version is applicable, the newest version at least as old as the oldest platform is selected, + * among all versions released for this system. If no such versions exists, throws an IllegalStateException. + */ + private Version compileVersion(ApplicationId id) { + Version oldestPlatform = controller.applications().oldestInstalledPlatform(id); + return controller.versionStatus().versions().stream() + .filter(VespaVersion::isReleased) + .map(VespaVersion::versionNumber) + .filter(version -> ! version.isAfter(oldestPlatform)) + .max(Comparator.naturalOrder()) + .orElseGet(() -> controller.mavenRepository().metadata().versions().stream() + .filter(version -> ! version.isAfter(oldestPlatform)) + .max(Comparator.naturalOrder()) + .orElseThrow(() -> new IllegalStateException("No available releases of " + + controller.mavenRepository().artifactId()))); + } + private HttpResponse setGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region, boolean inService, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); ZoneId zone = ZoneId.from(environment, region); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java index 871cc4d58cc..161c625ce16 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -29,7 +29,7 @@ public class MavenRepositoryClient implements MavenRepository { } @Override - public Metadata getMetadata() { + public Metadata metadata() { try { HttpRequest request = HttpRequest.newBuilder(withArtifactPath(apiUrl, id)).build(); HttpResponse response = client.send(request, HttpResponse.BodyHandlers.ofString(UTF_8)); @@ -44,6 +44,11 @@ public class MavenRepositoryClient implements MavenRepository { } } + @Override + public ArtifactId artifactId() { + return id; + } + static URI withArtifactPath(URI baseUrl, ArtifactId id) { List parts = new ArrayList<>(List.of(id.groupId().split("\\."))); parts.add(id.artifactId()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 6ff14730285..87f35d3b2c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -12,7 +12,6 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.github.GitSha; -import com.yahoo.vespa.hosted.controller.api.integration.maven.ArtifactId; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobList; @@ -130,7 +129,7 @@ public class VersionStatus { Collection deploymentStatistics = computeDeploymentStatistics(infrastructureVersions, controller.applications().asList()); List versions = new ArrayList<>(); - List releasedVersions = controller.mavenRepository().getMetadata().versions(); + List releasedVersions = controller.mavenRepository().metadata().versions(); for (DeploymentStatistics statistics : deploymentStatistics) { if (statistics.version().isEmpty()) continue; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index 46f18f9d813..bf4ae9663f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -76,7 +76,7 @@ } ], "changeBlockers": [], - "compileVersion": "6.1.0", + "compileVersion": "6.0.0", "majorVersion": 7, "globalRotations": [], "instances": [], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 3063bb62b7e..1d5a5aac089 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -76,7 +76,7 @@ } ], "changeBlockers": [], - "compileVersion": "6.1.0", + "compileVersion": "6.0.0", "globalRotations": [], "instances": [], "metrics": { -- cgit v1.2.3 From 03b939ab77553dca9af710945c0a32e0e4230b1f Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 14:35:16 +0200 Subject: FIlter out broken confidence versions for compile version --- .../controller/api/integration/stubs/MockMavenRepository.java | 1 + .../controller/restapi/application/ApplicationApiHandler.java | 7 ++++++- .../vespa/hosted/controller/restapi/ContainerControllerTester.java | 2 +- .../hosted/controller/restapi/application/ApplicationApiTest.java | 5 +++++ .../controller/restapi/application/responses/application.json | 2 +- .../restapi/application/responses/application2-with-patches.json | 2 +- .../controller/restapi/application/responses/application2.json | 2 +- .../hosted/controller/restapi/controller/ControllerApiTest.java | 2 +- 8 files changed, 17 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java index 0ecb7ae13cc..be1deb3997a 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/stubs/MockMavenRepository.java @@ -19,6 +19,7 @@ public class MockMavenRepository implements MavenRepository { @Override public Metadata metadata() { return new Metadata(id, List.of(Version.fromString("6.0"), + Version.fromString("6.1"), Version.fromString("6.2"))); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 79684e705b1..9f091061596 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -105,6 +105,7 @@ import java.util.Scanner; import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; +import java.util.stream.Collectors; import static java.util.stream.Collectors.joining; @@ -695,7 +696,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { } /** - * Returns a version at least as old as the oldest platform the given application is on, and which is released. + * Returns a non-broken, released version at least as old as the oldest platform the given application is on. * * If no known version is applicable, the newest version at least as old as the oldest platform is selected, * among all versions released for this system. If no such versions exists, throws an IllegalStateException. @@ -703,12 +704,16 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private Version compileVersion(ApplicationId id) { Version oldestPlatform = controller.applications().oldestInstalledPlatform(id); return controller.versionStatus().versions().stream() + .filter(version -> version.confidence().equalOrHigherThan(VespaVersion.Confidence.low)) .filter(VespaVersion::isReleased) .map(VespaVersion::versionNumber) .filter(version -> ! version.isAfter(oldestPlatform)) .max(Comparator.naturalOrder()) .orElseGet(() -> controller.mavenRepository().metadata().versions().stream() .filter(version -> ! version.isAfter(oldestPlatform)) + .filter(version -> ! controller.versionStatus().versions().stream() + .map(VespaVersion::versionNumber) + .collect(Collectors.toSet()).contains(version)) .max(Comparator.naturalOrder()) .orElseThrow(() -> new IllegalStateException("No available releases of " + controller.mavenRepository().artifactId()))); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java index 76c505ff8f8..427428a3a94 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerControllerTester.java @@ -54,7 +54,7 @@ public class ContainerControllerTester { public ContainerControllerTester(JDisc container, String responseFilePath) { containerTester = new ContainerTester(container, responseFilePath); - CuratorDb curatorDb = new MockCuratorDb(); + CuratorDb curatorDb = controller().curator(); curatorDb.writeUpgradesPerMinute(100); upgrader = new Upgrader(controller(), Duration.ofDays(1), new JobControl(curatorDb), curatorDb); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 71c4b41a276..16fd10277d2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -61,6 +61,8 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; +import com.yahoo.vespa.hosted.controller.versions.VersionStatus; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; import org.junit.Test; @@ -369,6 +371,9 @@ public class ApplicationApiTest extends ControllerContainerTest { .oktaAccessToken(OKTA_AT), ""); + // Set version 6.1 to broken to change compile version for. + controllerTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); + tester.computeVersionStatus(); setDeploymentMaintainedInfo(controllerTester); // GET tenant application deployments tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json index 5d1819bf0f2..1d719133ac3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json @@ -217,7 +217,7 @@ ] } ], - "compileVersion": "(ignore)", + "compileVersion": "6.0.0", "globalRotations": [ "https://application1--tenant1.global.vespa.oath.cloud:4443/" ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json index bf4ae9663f7..46f18f9d813 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2-with-patches.json @@ -76,7 +76,7 @@ } ], "changeBlockers": [], - "compileVersion": "6.0.0", + "compileVersion": "6.1.0", "majorVersion": 7, "globalRotations": [], "instances": [], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json index 1d5a5aac089..3063bb62b7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application2.json @@ -76,7 +76,7 @@ } ], "changeBlockers": [], - "compileVersion": "6.0.0", + "compileVersion": "6.1.0", "globalRotations": [], "instances": [], "metrics": { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java index ba81c5cf4e4..74d637499bd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java @@ -80,7 +80,7 @@ public class ControllerApiTest extends ControllerContainerTest { public void testUpgraderApi() { // Get current configuration tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/jobs/upgrader", new byte[0], Request.Method.GET), - "{\"upgradesPerMinute\":0.125,\"confidenceOverrides\":[]}", + "{\"upgradesPerMinute\":100.0,\"confidenceOverrides\":[]}", 200); // Set invalid configuration -- cgit v1.2.3 From cc95857da66d1d924f81ed93a15b44ce38626a4d Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 16:08:10 +0200 Subject: @author and comment --- .../vespa/hosted/controller/versions/MavenRepositoryClient.java | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java index 161c625ce16..9f3addd4992 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/MavenRepositoryClient.java @@ -16,6 +16,11 @@ import java.util.List; import static java.nio.charset.StandardCharsets.UTF_8; +/** + * Http client implementation of a {@link MavenRepository}, which uses a configured repository and artifact ID. + * + * @author jonmv + */ public class MavenRepositoryClient implements MavenRepository { private final HttpClient client; -- cgit v1.2.3 From 2dc7fd60f34e2bcb61dd326da9de53157301526c Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 20 Jun 2019 16:18:56 +0200 Subject: Default to all versions being released, as now --- .../vespa/hosted/controller/persistence/VersionStatusSerializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java index a4b339a69bc..207a5f8dcf9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/VersionStatusSerializer.java @@ -107,7 +107,7 @@ public class VersionStatusSerializer { Instant.ofEpochMilli(object.field(committedAtField).asLong()), object.field(isControllerVersionField).asBool(), object.field(isSystemVersionField).asBool(), - object.field(isReleasedField).asBool(), + object.field(isReleasedField).valid() ? object.field(isReleasedField).asBool() : true, configServersFromSlime(object.field(configServersField)), VespaVersion.Confidence.valueOf(object.field(confidenceField).asString()) ); -- cgit v1.2.3 From 1233ea296dec76cec028067409ce5b713aac693a Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Thu, 20 Jun 2019 22:55:40 +0200 Subject: Export package --- .../vespa/hosted/controller/api/integration/maven/package-info.java | 5 +++++ .../com/yahoo/vespa/hosted/controller/versions/package-info.java | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java new file mode 100644 index 00000000000..d5abdf31f4b --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/maven/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.api.integration.maven; + +import com.yahoo.osgi.annotation.ExportPackage; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java new file mode 100644 index 00000000000..c6f2c1e427d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/package-info.java @@ -0,0 +1,5 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +@ExportPackage +package com.yahoo.vespa.hosted.controller.versions; + +import com.yahoo.osgi.annotation.ExportPackage; -- cgit v1.2.3 From 401fed1e55ce5ccc6cef75b153efd83ca410c824 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 20 Jun 2019 16:56:06 +0200 Subject: Always refresh routing policies --- .../hosted/controller/ApplicationController.java | 19 ++++++++++--------- .../controller/maintenance/RoutingPoliciesTest.java | 1 - 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 7b03664e0ab..782a76b684e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -422,15 +422,16 @@ public class ApplicationController { ZoneId zone, DeployOptions deployOptions, Set rotationNames) { DeploymentId deploymentId = new DeploymentId(application, zone); - ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationPackage.zippedContent()); - - // Refresh routing policies on successful deployment. At this point we can safely assume that the config server - // has allocated load balancers for the deployment. - routingPolicies.refresh(application, zone); - - return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), - applicationPackage.zippedContent().length); + try { + ConfigServer.PreparedApplication preparedApplication = + configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationPackage.zippedContent()); + return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), + applicationPackage.zippedContent().length); + } finally { + // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that + // any DNS updates can be propagated as early as possible. + routingPolicies.refresh(application, zone); + } } /** Makes sure the application has a global rotation, if eligible. */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java index 449ca509ee4..f0344cb8d12 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java @@ -11,7 +11,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -- cgit v1.2.3 From e37a674b61ca298092a333b2677f1c36cc061ec9 Mon Sep 17 00:00:00 2001 From: Håkon Hallingstad Date: Mon, 24 Jun 2019 11:20:13 +0200 Subject: Avoid too much use of ZoneList.ids() now that zones() is available --- .../java/com/yahoo/config/provision/zone/ZoneList.java | 9 +++++++-- .../controller/maintenance/ResourceMeterMaintainer.java | 5 +++-- .../controller/proxy/ConfigServerRestExecutorImpl.java | 5 +++-- .../hosted/controller/restapi/cost/CostCalculator.java | 5 +++-- .../hosted/controller/restapi/os/OsApiHandler.java | 4 +++- .../controller/restapi/zone/v1/ZoneApiHandler.java | 13 +++++++------ .../controller/restapi/zone/v2/ZoneApiHandler.java | 12 ++++++------ .../vespa/hosted/controller/versions/VersionStatus.java | 10 ++++------ .../hosted/controller/deployment/DeploymentTester.java | 7 ++++--- .../hosted/controller/integration/ZoneFilterMock.java | 5 ----- .../hosted/controller/restapi/ContainerTester.java | 7 ++++--- .../hosted/controller/versions/VersionStatusTest.java | 17 +++++++++-------- 12 files changed, 53 insertions(+), 46 deletions(-) (limited to 'controller-server') diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java index 5f3f2e10898..776f925c424 100644 --- a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java +++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/ZoneList.java @@ -1,10 +1,13 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.config.provision.zone; +import com.google.common.collect.ImmutableList; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; import java.util.List; +import java.util.stream.Collectors; /** * Provides filters for and access to a list of ZoneIds. @@ -32,7 +35,9 @@ public interface ZoneList extends ZoneFilter { /** Returns the ZoneApi of all zones in this list. */ List zones(); - /** Returns the id of all zones in this list as — you guessed it — a list. */ - List ids(); + /** Returns the ZoneIds of all zones in this list. */ + default List ids() { + return zones().stream().map(ZoneApi::getId).collect(Collectors.toList()); + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index 9302ecbe738..c4f0597572b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.config.provision.ApplicationId; @@ -81,8 +82,8 @@ public class ResourceMeterMaintainer extends Maintainer { private List getNodes() { return controller().zoneRegistry().zones() .ofCloud(CloudName.from("aws")) - .reachable().ids().stream() - .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) + .reachable().zones().stream() + .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId(), true).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .filter(node -> node.getState() == NodeState.active) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java index a208249b410..73a029ad3b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/proxy/ConfigServerRestExecutorImpl.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Inject; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.config.provision.zone.ZoneList; import com.yahoo.jdisc.http.HttpRequest.Method; @@ -114,9 +115,9 @@ public class ConfigServerRestExecutorImpl implements ConfigServerRestExecutor { if ( ! environmentName.isEmpty()) zones = zones.in(Environment.from(environmentName)); - for (ZoneId zoneId : zones.ids()) { + for (ZoneApi zone : zones.zones()) { responseStructure.uris.add(proxyRequest.getScheme() + "://" + proxyRequest.getControllerPrefix() + - zoneId.environment().value() + "/" + zoneId.region().value()); + zone.getEnvironment().value() + "/" + zone.getRegionName().value()); } JsonNode node = mapper.valueToTree(responseStructure); return new ProxyResponse(proxyRequest, node.toString(), 200, Optional.empty(), "application/json"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index 18c00d69b62..c44a80f7a20 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; @@ -34,8 +35,8 @@ public class CostCalculator { String date = LocalDate.now(clock).toString(); List nodes = controller.zoneRegistry().zones() - .reachable().in(Environment.prod).ofCloud(cloudName).ids().stream() - .flatMap(zoneId -> uncheck(() -> nodeRepository.listNodes(zoneId, true).nodes().stream())) + .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream() + .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId(), true).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java index 5454d71185a..bc360fe3c6f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/os/OsApiHandler.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.io.IOUtils; @@ -30,6 +31,7 @@ import java.util.List; import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; +import java.util.stream.Collectors; /** * This implements the /os/v1 API which provides operators with information about, and scheduling of OS upgrades for @@ -123,7 +125,7 @@ public class OsApiHandler extends AuditLoggingRequestHandler { ZoneList zones = controller.zoneRegistry().zones().controllerUpgraded(); if (path.get("region") != null) zones = zones.in(RegionName.from(path.get("region"))); if (path.get("environment") != null) zones = zones.in(Environment.from(path.get("environment"))); - return zones.ids(); + return zones.zones().stream().map(ZoneApi::getId).collect(Collectors.toList()); } private Slime setOsVersion(HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java index b115e659c28..6cfaed93fa9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v1/ZoneApiHandler.java @@ -3,6 +3,8 @@ package com.yahoo.vespa.hosted.controller.restapi.zone.v1; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.Zone; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; @@ -70,8 +72,8 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse root(HttpRequest request) { - List environments = zoneRegistry.zones().all().ids().stream() - .map(ZoneId::environment) + List environments = zoneRegistry.zones().all().zones().stream() + .map(ZoneApi::getEnvironment) .distinct() .sorted(Comparator.comparing(Environment::value)) .collect(Collectors.toList()); @@ -90,17 +92,16 @@ public class ZoneApiHandler extends LoggingRequestHandler { } private HttpResponse environment(HttpRequest request, Environment environment) { - List zones = zoneRegistry.zones().all().in(environment).ids(); Slime slime = new Slime(); Cursor root = slime.setArray(); - zones.forEach(zone -> { + zoneRegistry.zones().all().in(environment).zones().forEach(zone -> { Cursor object = root.addObject(); - object.setString("name", zone.region().value()); + object.setString("name", zone.getRegionName().value()); object.setString("url", request.getUri() .resolve("/zone/v2/environment/") .resolve(environment.value() + "/") .resolve("region/") - .resolve(zone.region().value()) + .resolve(zone.getRegionName().value()) .toString()); }); return new SlimeJsonResponse(slime); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java index 9d95383fbfb..f0259fc4d51 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/zone/v2/ZoneApiHandler.java @@ -94,16 +94,16 @@ public class ZoneApiHandler extends AuditLoggingRequestHandler { Cursor root = slime.setObject(); Cursor uris = root.setArray("uris"); ZoneList zoneList = zoneRegistry.zones().reachable(); - zoneList.ids().forEach(zoneId -> uris.addString(request.getUri() + zoneList.zones().forEach(zone -> uris.addString(request.getUri() .resolve("/zone/v2/") - .resolve(zoneId.environment().value() + "/") - .resolve(zoneId.region().value()) + .resolve(zone.getEnvironment().value() + "/") + .resolve(zone.getRegionName().value()) .toString())); Cursor zones = root.setArray("zones"); - zoneList.ids().forEach(zoneId -> { + zoneList.zones().forEach(zone -> { Cursor object = zones.addObject(); - object.setString("environment", zoneId.environment().value()); - object.setString("region", zoneId.region().value()); + object.setString("environment", zone.getEnvironment().value()); + object.setString("region", zone.getRegionName().value()); }); return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java index 87f35d3b2c1..ab5fd2714e5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VersionStatus.java @@ -6,6 +6,7 @@ import com.yahoo.collections.ListMap; import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.log.LogLevel; import com.yahoo.vespa.hosted.controller.Application; @@ -155,20 +156,17 @@ public class VersionStatus { } private static ListMap findSystemApplicationVersions(Controller controller) { - List zones = controller.zoneRegistry().zones() - .controllerUpgraded() - .ids(); ListMap versions = new ListMap<>(); - for (ZoneId zone : zones) { + for (ZoneApi zone : controller.zoneRegistry().zones().controllerUpgraded().zones()) { for (SystemApplication application : SystemApplication.all()) { List eligibleForUpgradeApplicationNodes = controller.configServer().nodeRepository() - .list(zone, application.id()).stream() + .list(zone.getId(), application.id()).stream() .filter(SystemUpgrader::eligibleForUpgrade) .collect(Collectors.toList()); if (eligibleForUpgradeApplicationNodes.isEmpty()) continue; - boolean configConverged = application.configConvergedIn(zone, controller, Optional.empty()); + boolean configConverged = application.configConvergedIn(zone.getId(), controller, Optional.empty()); if (!configConverged) { log.log(LogLevel.WARNING, "Config for " + application.id() + " in " + zone + " has not converged"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java index 3ce32347e35..887406ecba8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTester.java @@ -5,6 +5,7 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.Application; @@ -123,10 +124,10 @@ public class DeploymentTester { /** Upgrade system applications in all zones to given version */ public void upgradeSystemApplications(Version version) { - for (ZoneId zone : tester.zoneRegistry().zones().all().ids()) { + for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { for (SystemApplication application : SystemApplication.all()) { - tester.configServer().setVersion(application.id(), zone, version); - tester.configServer().convergeServices(application.id(), zone); + tester.configServer().setVersion(application.id(), zone.getId(), version); + tester.configServer().convergeServices(application.id(), zone.getId()); } } computeVersionStatus(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java index 57f29fb72af..00e6162d5e5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java @@ -81,11 +81,6 @@ public class ZoneFilterMock implements ZoneList { return List.copyOf(zones); } - @Override - public List ids() { - return List.copyOf(zones.stream().map(ZoneApi::getId).collect(Collectors.toList())); - } - @Override public ZoneList ofCloud(CloudName cloud) { return filter(zone -> zone.getCloudName().equals(cloud)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java index ef86ffa125f..c7be543dd00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java @@ -6,6 +6,7 @@ import com.yahoo.application.container.handler.Request; import com.yahoo.application.container.handler.Response; import com.yahoo.component.ComponentSpecification; import com.yahoo.component.Version; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.http.filter.FilterChainRepository; import com.yahoo.jdisc.http.filter.SecurityRequestFilter; @@ -59,10 +60,10 @@ public class ContainerTester { public void upgradeSystem(Version version) { controller().curator().writeControllerVersion(controller().hostname(), version); - for (ZoneId zone : controller().zoneRegistry().zones().all().ids()) { + for (ZoneApi zone : controller().zoneRegistry().zones().all().zones()) { for (SystemApplication application : SystemApplication.all()) { - configServer().setVersion(application.id(), zone, version); - configServer().convergeServices(application.id(), zone); + configServer().setVersion(application.id(), zone.getId(), version); + configServer().convergeServices(application.id(), zone.getId()); } } computeVersionStatus(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java index 8e3dc24193f..655c16ccceb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/versions/VersionStatusTest.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.component.Vtag; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -60,10 +61,10 @@ public class VersionStatusTest { Version version0 = Version.fromString("6.1"); Version version1 = Version.fromString("6.5"); // Upgrade some config servers - for (ZoneId zone : tester.zoneRegistry().zones().all().ids()) { - for (Node node : tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id())) { - tester.configServer().nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), - node.owner(), version1, node.wantedVersion())); + for (ZoneApi zone : tester.zoneRegistry().zones().all().zones()) { + for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { + Node upgradedNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), version1, node.wantedVersion()); + tester.configServer().nodeRepository().putByHostname(zone.getId(), upgradedNode); break; } } @@ -105,10 +106,10 @@ public class VersionStatusTest { // Downgrade one config server in each zone Version ancientVersion = Version.fromString("5.1"); - for (ZoneId zone : tester.controller().zoneRegistry().zones().all().ids()) { - for (Node node : tester.configServer().nodeRepository().list(zone, SystemApplication.configServer.id())) { - tester.configServer().nodeRepository().putByHostname(zone, new Node(node.hostname(), node.state(), node.type(), - node.owner(), ancientVersion, node.wantedVersion())); + for (ZoneApi zone : tester.controller().zoneRegistry().zones().all().zones()) { + for (Node node : tester.configServer().nodeRepository().list(zone.getId(), SystemApplication.configServer.id())) { + Node downgradedNode = new Node(node.hostname(), node.state(), node.type(), node.owner(), ancientVersion, node.wantedVersion()); + tester.configServer().nodeRepository().putByHostname(zone.getId(), downgradedNode); break; } } -- cgit v1.2.3 From 4b56d4b6dfa9ae11141f7503f14264c7e1378bb1 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 25 Jun 2019 12:32:02 +0200 Subject: Support AssignedRotations that track cluster, rotation, and endpoint name. No functionality changes in this commit, only how we persist and talk about rotations and endpoints in the applicaiton. This is preparation of actually updating the support for multiple endpoints in the controller. --- .../yahoo/vespa/hosted/controller/Application.java | 29 +++++++--- .../hosted/controller/ApplicationController.java | 4 +- .../vespa/hosted/controller/LockedApplication.java | 53 +++++++++---------- .../controller/application/AssignedRotation.java | 61 ++++++++++++++++++++++ .../hosted/controller/application/EndpointId.java | 47 +++++++++++++++++ .../persistence/ApplicationSerializer.java | 60 +++++++++++++++++---- .../persistence/ApplicationSerializerTest.java | 11 ++-- 7 files changed, 214 insertions(+), 51 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index b4e3b8b1a9a..0a85e68f4b6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -5,9 +5,11 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; @@ -15,6 +17,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationV import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.vespa.hosted.controller.application.ApplicationActivity; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; @@ -27,14 +30,17 @@ import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.Collections; import java.util.Comparator; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; /** * An instance of an application. @@ -58,8 +64,7 @@ public class Application { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional pemDeployKey; - private final Optional legacyRotation; - private final List rotations; + private final List rotations; private final Map rotationStatus; /** Creates an empty application */ @@ -68,7 +73,7 @@ public class Application { new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), - Optional.empty(), Optional.empty(), Collections.emptyList(), Collections.emptyMap()); + Optional.empty(), Collections.emptyList(), Collections.emptyMap()); } /** Used from persistence layer: Do not use */ @@ -76,18 +81,18 @@ public class Application { List deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - Optional legacyRotation, List rotations, Map rotationStatus) { + List rotations, Map rotationStatus) { this(id, createdAt, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())), deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } Application(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - Optional legacyRotation, List rotations, Map rotationStatus) { + List rotations, Map rotationStatus) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -101,7 +106,6 @@ public class Application { this.majorVersion = Objects.requireNonNull(majorVersion, "majorVersion cannot be null"); this.metrics = Objects.requireNonNull(metrics, "metrics cannot be null"); this.pemDeployKey = pemDeployKey; - this.legacyRotation = Objects.requireNonNull(legacyRotation, "legacyRotation cannot be null"); this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); } @@ -200,11 +204,20 @@ public class Application { /** Returns the global rotation id of this, if present */ public Optional legacyRotation() { - return legacyRotation; + return rotations.stream() + .map(AssignedRotation::rotationId) + .findFirst(); } /** Returns all rotations for this application */ public List rotations() { + return rotations.stream() + .map(AssignedRotation::rotationId) + .collect(Collectors.toList()); + } + + /** Returns all assigned rotations for this application */ + public List assignedRotations() { return rotations; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 782a76b684e..8345aa91685 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -43,9 +43,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; @@ -439,7 +441,7 @@ public class ApplicationController { if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) { try (RotationLock rotationLock = rotationRepository.lock()) { Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock); - application = application.with(rotation.id()); + application = application.with(List.of(new AssignedRotation(new ClusterSpec.Id(application.get().deploymentSpec().globalServiceId().get()), EndpointId.default_(), rotation.id()))); store(application); // store assigned rotation even if deployment fails boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 5f958b74c39..42162784957 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -56,8 +57,7 @@ public class LockedApplication { private final OptionalInt majorVersion; private final ApplicationMetrics metrics; private final Optional pemDeployKey; - private final Optional legacyRotation; - private final List rotations; + private final List rotations; private final Map rotationStatus; /** @@ -72,7 +72,7 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), - application.pemDeployKey(), application.legacyRotation(), application.rotations(), application.rotationStatus()); + application.pemDeployKey(), application.assignedRotations(), application.rotationStatus()); } private LockedApplication(Lock lock, ApplicationId id, Instant createdAt, @@ -80,7 +80,7 @@ public class LockedApplication { Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - Optional legacyRotation, List rotations, Map rotationStatus) { + List rotations, Map rotationStatus) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -95,7 +95,6 @@ public class LockedApplication { this.majorVersion = majorVersion; this.metrics = metrics; this.pemDeployKey = pemDeployKey; - this.legacyRotation = legacyRotation; this.rotations = rotations; this.rotationStatus = rotationStatus; } @@ -104,35 +103,35 @@ public class LockedApplication { public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.with(issueId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, @@ -140,14 +139,14 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withCompletion(projectId, jobType, completion, jobError), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, legacyRotation, rotations, rotationStatus); + pemDeployKey, rotations, rotationStatus); } public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withTriggering(jobType, job), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -198,45 +197,45 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, Optional.ofNullable(issueId), owner, - majorVersion, metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + majorVersion, metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, Optional.ofNullable(owner), majorVersion, metrics, pemDeployKey, - legacyRotation, rotations, rotationStatus); + rotations, rotationStatus); } /** Set a major version for this, or set to null to remove any major version override */ @@ -244,31 +243,31 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication with(MetricsService.ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } public LockedApplication withPemDeployKey(String pemDeployKey) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, Optional.ofNullable(pemDeployKey), legacyRotation, rotations, rotationStatus); + metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus); } - public LockedApplication with(RotationId rotation) { + public LockedApplication with(List assignedRotations) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, - deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, Optional.of(rotation), List.of(rotation), rotationStatus); + deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, + metrics, pemDeployKey, assignedRotations, rotationStatus); } public LockedApplication withRotationStatus(Map rotationStatus) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } /** Don't expose non-leaf sub-objects. */ @@ -281,7 +280,7 @@ public class LockedApplication { private LockedApplication with(Map deployments) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, legacyRotation, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java new file mode 100644 index 00000000000..e1ed278a79e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -0,0 +1,61 @@ +package com.yahoo.vespa.hosted.controller.application; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.hosted.controller.rotation.RotationId; + +import java.util.Objects; + +/** + * Contains the tuple of [clusterId, endpointId, rotationId], to keep track + * of which services have assigned which rotations under which name. + * + * @author ogronnesby + */ +public class AssignedRotation { + private final ClusterSpec.Id clusterId; + private final EndpointId endpointId; + private final RotationId rotationId; + + public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId) { + this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId"); + this.endpointId = Objects.requireNonNull(endpointId); + this.rotationId = Objects.requireNonNull(rotationId); + } + + public ClusterSpec.Id clusterId() { return clusterId; } + public EndpointId endpointId() { return endpointId; } + public RotationId rotationId() { return rotationId; } + + @Override + public String toString() { + return "AssignedRotation{" + + "clusterId=" + clusterId + + ", endpointId='" + endpointId + '\'' + + ", rotationId=" + rotationId + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + AssignedRotation that = (AssignedRotation) o; + return clusterId.equals(that.clusterId) && + endpointId.equals(that.endpointId) && + rotationId.equals(that.rotationId); + } + + @Override + public int hashCode() { + return Objects.hash(clusterId, endpointId, rotationId); + } + + private static T requireNonEmpty(T object, String value, String field) { + Objects.requireNonNull(object); + Objects.requireNonNull(value); + if (value.isEmpty()) { + throw new IllegalArgumentException("Field '" + field + "' was empty"); + } + return object; + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java new file mode 100644 index 00000000000..f186125d09a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java @@ -0,0 +1,47 @@ +package com.yahoo.vespa.hosted.controller.application; + +import java.util.Objects; + +public class EndpointId { + private static final EndpointId DEFAULT = new EndpointId("default"); + + private final String id; + + public EndpointId(String id) { + this.id = requireNotEmpty(id); + } + + public String id() { return id; } + + @Override + public String toString() { + return "EndpointId{" + + "id='" + id + '\'' + + '}'; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + EndpointId that = (EndpointId) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + private static String requireNotEmpty(String input) { + Objects.requireNonNull(input); + if (input.isEmpty()) { + throw new IllegalArgumentException("The value EndpointId was empty"); + } + return input; + } + + public static EndpointId default_() { return DEFAULT; } + + public static EndpointId of(String id) { return new EndpointId(id); } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 1f20bdf5533..949b7d000d9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.document.datatypes.Array; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -21,6 +22,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -29,6 +31,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -38,6 +41,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,6 +81,7 @@ public class ApplicationSerializer { private final String writeQualityField = "writeQuality"; private final String queryQualityField = "queryQuality"; private final String pemDeployKeyField = "pemDeployKey"; + private final String assignedRotationsField = "assignedRotations"; private final String rotationsField = "endpoints"; private final String deprecatedRotationField = "rotation"; private final String rotationStatusField = "rotationStatus"; @@ -171,8 +176,8 @@ public class ApplicationSerializer { root.setDouble(writeQualityField, application.metrics().writeServiceQuality()); application.pemDeployKey().ifPresent(pemDeployKey -> root.setString(pemDeployKeyField, pemDeployKey)); application.legacyRotation().ifPresent(rotation -> root.setString(deprecatedRotationField, rotation.asString())); - Cursor rotations = root.setArray(rotationsField); - application.rotations().forEach(rotation -> rotations.addString(rotation.asString())); + rotationsToSlime(application.assignedRotations(), root, rotationsField); + assignedRotationsToSlime(application.assignedRotations(), root, assignedRotationsField); toSlime(application.rotationStatus(), root.setArray(rotationStatusField)); return slime; } @@ -320,6 +325,21 @@ public class ApplicationSerializer { }); } + private void rotationsToSlime(List rotations, Cursor parent, String fieldName) { + final var rotationsArray = parent.setArray(fieldName); + rotations.forEach(rot -> rotationsArray.addString(rot.rotationId().asString())); + } + + private void assignedRotationsToSlime(List rotations, Cursor parent, String fieldName) { + final var rotationsArray = parent.setArray(fieldName); + for (var rotation : rotations) { + final var object = rotationsArray.addObject(); + object.setString("endpoint", rotation.endpointId().id()); + object.setString("rotation", rotation.rotationId().asString()); + object.setString("container", rotation.clusterId().value()); + } + } + // ------------------ Deserialization public Application fromSlime(Slime slime) { @@ -339,13 +359,12 @@ public class ApplicationSerializer { ApplicationMetrics metrics = new ApplicationMetrics(root.field(queryQualityField).asDouble(), root.field(writeQualityField).asDouble()); Optional pemDeployKey = optionalString(root.field(pemDeployKeyField)); - Optional legacyRotation = optionalString(root.field(deprecatedRotationField)).map(RotationId::new); - List rotations = rotationsFromSlime(root); + List assignedRotations = assignedRotationsFromSlime(deploymentSpec, root); Map rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField)); return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, legacyRotation, rotations, rotationStatus); + pemDeployKey, assignedRotations, rotationStatus); } private List deploymentsFromSlime(Inspector array) { @@ -525,15 +544,36 @@ public class ApplicationSerializer { Instant.ofEpochMilli(object.field(atField).asLong()))); } - private List rotationsFromSlime(Inspector root) { - final var rotations = rotationListFromSlime(root.field(rotationsField)); + private List assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) { + final var assignedRotations = new LinkedHashSet(); + + // Add the legacy rotation field to the set - this needs to be first + // TODO: Remove when we retire the rotations field final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); + if (legacyRotation.isPresent() && deploymentSpec.globalServiceId().isPresent()) { + final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); + assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get())); + } - if (legacyRotation.isPresent() && ! rotations.contains(legacyRotation.get())) { - rotations.add(legacyRotation.get()); + // Now add the same entries from "stupid" list of rotations + // TODO: Remove when we retire the rotations field + final var rotations = rotationListFromSlime(root.field(rotationsField)); + for (var rotation : rotations) { + if (deploymentSpec.globalServiceId().isPresent()) { + final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); + assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), rotation)); + } } - return rotations; + // Last - add the actual entries we want. Do _not_ remove this during clean-up + root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> { + final var clusterId = new ClusterSpec.Id(inspector.field("container").asString()); + final var endpointId = EndpointId.of(inspector.field("endpoint").asString()); + final var rotationId = new RotationId(inspector.field("rotation").asString()); + assignedRotations.add(new AssignedRotation(clusterId, endpointId, rotationId)); + }); + + return List.copyOf(assignedRotations); } private List rotationListFromSlime(Inspector field) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index be9624fc693..de08f1e3576 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.ClusterUtilization; @@ -24,6 +25,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.JobStatus; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -116,8 +118,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), - Optional.of(new RotationId("my-rotation")), - List.of(new RotationId("my-rotation")), + List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"))), rotationStatus); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -266,9 +267,9 @@ public class ApplicationSerializerTest { assertEquals( List.of( - new RotationId("multiple-rotation-1"), - new RotationId("multiple-rotation-2"), - new RotationId("single-rotation") + new RotationId("single-rotation"), + new RotationId("multiple-rotation-1"), + new RotationId("multiple-rotation-2") ), application.rotations() ); -- cgit v1.2.3 From fe27d5886c030cf94ee5580ae937da86d86e1e2c Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 25 Jun 2019 12:36:22 +0200 Subject: add javadoc and clean up unused imports --- .../main/java/com/yahoo/vespa/hosted/controller/Application.java | 7 ------- .../com/yahoo/vespa/hosted/controller/application/EndpointId.java | 6 ++++++ .../vespa/hosted/controller/persistence/ApplicationSerializer.java | 1 - .../vespa/hosted/controller/rotation/RotationRepositoryTest.java | 1 - 4 files changed, 6 insertions(+), 9 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 0a85e68f4b6..c168d09a15a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -5,11 +5,9 @@ import com.google.common.collect.ImmutableMap; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; -import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RotationName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; @@ -21,26 +19,21 @@ import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; -import com.yahoo.vespa.hosted.controller.rotation.Rotation; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.time.Instant; import java.util.Collections; import java.util.Comparator; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalInt; import java.util.OptionalLong; -import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * An instance of an application. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java index f186125d09a..13c242c7b5f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java @@ -2,6 +2,12 @@ package com.yahoo.vespa.hosted.controller.application; import java.util.Objects; +/** + * A type to represent the ID of an endpoint. This is typically the first part of + * an endpoint name. + * + * @author ogronnesby + */ public class EndpointId { private static final EndpointId DEFAULT = new EndpointId("default"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 949b7d000d9..727a0e05b7f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; -import com.yahoo.document.datatypes.Array; import com.yahoo.slime.ArrayTraverser; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java index 02a82e35f10..8f02fa74c6e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java @@ -15,7 +15,6 @@ import org.junit.rules.ExpectedException; import java.net.URI; import java.util.List; -import java.util.Optional; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; -- cgit v1.2.3 From 1be2f4bd056f46ccc46c29db5de3775e6a0de744 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 25 Jun 2019 13:39:43 +0200 Subject: Use static variables for assigned rotation fields --- .../controller/persistence/ApplicationSerializer.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 727a0e05b7f..271221e15b2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -81,6 +81,9 @@ public class ApplicationSerializer { private final String queryQualityField = "queryQuality"; private final String pemDeployKeyField = "pemDeployKey"; private final String assignedRotationsField = "assignedRotations"; + private final String assignedRotationEndpointField = "endpointId"; + private final String assignedRotationClusterField = "clusterId"; + private final String assignedRotationRotationField = "rotationId"; private final String rotationsField = "endpoints"; private final String deprecatedRotationField = "rotation"; private final String rotationStatusField = "rotationStatus"; @@ -333,9 +336,9 @@ public class ApplicationSerializer { final var rotationsArray = parent.setArray(fieldName); for (var rotation : rotations) { final var object = rotationsArray.addObject(); - object.setString("endpoint", rotation.endpointId().id()); - object.setString("rotation", rotation.rotationId().asString()); - object.setString("container", rotation.clusterId().value()); + object.setString(assignedRotationEndpointField, rotation.endpointId().id()); + object.setString(assignedRotationRotationField, rotation.rotationId().asString()); + object.setString(assignedRotationClusterField, rotation.clusterId().value()); } } @@ -566,9 +569,9 @@ public class ApplicationSerializer { // Last - add the actual entries we want. Do _not_ remove this during clean-up root.field(assignedRotationsField).traverse((ArrayTraverser) (idx, inspector) -> { - final var clusterId = new ClusterSpec.Id(inspector.field("container").asString()); - final var endpointId = EndpointId.of(inspector.field("endpoint").asString()); - final var rotationId = new RotationId(inspector.field("rotation").asString()); + final var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); + final var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); + final var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); assignedRotations.add(new AssignedRotation(clusterId, endpointId, rotationId)); }); -- cgit v1.2.3 From 5de7ab90d4e12ee3406377dc0ae2993492a15db2 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 25 Jun 2019 13:41:22 +0200 Subject: Manually indent constructor parameters --- .../java/com/yahoo/vespa/hosted/controller/LockedApplication.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 42162784957..02b0afdd48f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -260,8 +260,8 @@ public class LockedApplication { public LockedApplication with(List assignedRotations) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, - deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, assignedRotations, rotationStatus); + deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, + metrics, pemDeployKey, assignedRotations, rotationStatus); } public LockedApplication withRotationStatus(Map rotationStatus) { -- cgit v1.2.3 From 22779d33769d9e1021da0789cd1c31c7e13ab149 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 25 Jun 2019 14:55:31 +0200 Subject: Improve serialization test coverage for AssignedRotation --- .../persistence/ApplicationSerializerTest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index de08f1e3576..67b6b1ac61c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -262,6 +262,12 @@ public class ApplicationSerializerTest { rotations.addString("multiple-rotation-1"); rotations.addString("multiple-rotation-2"); + final var assignedRotations = cursor.setArray("assignedRotations"); + final var assignedRotation = assignedRotations.addObject(); + assignedRotation.setString("clusterId", "foobar"); + assignedRotation.setString("endpointId", "nice-endpoint"); + assignedRotation.setString("rotationId", "assigned-rotation"); + // Parse and test the output from parsing contains both legacy rotation and multiple rotations final var application = applicationSerializer.fromSlime(slime); @@ -269,7 +275,8 @@ public class ApplicationSerializerTest { List.of( new RotationId("single-rotation"), new RotationId("multiple-rotation-1"), - new RotationId("multiple-rotation-2") + new RotationId("multiple-rotation-2"), + new RotationId("assigned-rotation") ), application.rotations() ); @@ -277,6 +284,16 @@ public class ApplicationSerializerTest { assertEquals( Optional.of(new RotationId("single-rotation")), application.legacyRotation() ); + + assertEquals( + List.of( + new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation")), + new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-1")), + new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-2")), + new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation")) + ), + application.assignedRotations() + ); } @Test -- cgit v1.2.3 From 32e6286798fb36f6420fc718427def17dd3a1398 Mon Sep 17 00:00:00 2001 From: Morten Tokle Date: Wed, 12 Jun 2019 14:34:42 +0200 Subject: Request certificates for whitelisted apps. Send certificate reference with deploy --- .../certificates/ApplicationCertificate.java | 31 ++++++++----- .../ApplicationCertificateProvider.java | 12 +++++ .../certificates/CertificateProvider.java | 14 ------ .../api/integration/certificates/KeyId.java | 18 -------- .../integration/certificates/KeyPairProvider.java | 14 ------ .../integration/certificates/VersionedKeyPair.java | 28 ----------- .../api/integration/configserver/ConfigServer.java | 3 +- .../yahoo/vespa/hosted/controller/Application.java | 16 +++++-- .../hosted/controller/ApplicationController.java | 34 ++++++++++++-- .../yahoo/vespa/hosted/controller/Controller.java | 14 ++++-- .../vespa/hosted/controller/LockedApplication.java | 54 +++++++++++++--------- .../persistence/ApplicationSerializer.java | 6 ++- .../vespa/hosted/controller/ControllerTester.java | 6 ++- .../integration/ApplicationCertificateMock.java | 14 ++++++ .../controller/integration/ConfigServerMock.java | 3 +- .../persistence/ApplicationSerializerTest.java | 6 ++- .../restapi/ControllerContainerTest.java | 1 + .../src/main/java/com/yahoo/vespa/flags/Flags.java | 5 ++ 18 files changed, 152 insertions(+), 127 deletions(-) create mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/CertificateProvider.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyId.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyPairProvider.java delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/VersionedKeyPair.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationCertificateMock.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java index e4d0c8246d9..dbcb44d1711 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificate.java @@ -1,29 +1,36 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.api.integration.certificates; -import java.security.cert.X509Certificate; -import java.util.List; +import java.util.Objects; /** - * Represents a certificate chain and a reference to the private key used for generating the certificate + * Represents a reference to a certificate and private key. * * @author mortent * @author andreer */ public class ApplicationCertificate { - private final List certificateChain; - private final KeyId keyId; - public ApplicationCertificate(List certificateChain, KeyId keyId) { - this.certificateChain = certificateChain; - this.keyId = keyId; + private final String secretsKeyNamePrefix; + + public ApplicationCertificate(String secretsKeyNamePrefix) { + this.secretsKeyNamePrefix = secretsKeyNamePrefix; + } + + public String secretsKeyNamePrefix() { + return secretsKeyNamePrefix; } - public List certificateChain() { - return certificateChain; + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ApplicationCertificate that = (ApplicationCertificate) o; + return Objects.equals(secretsKeyNamePrefix, that.secretsKeyNamePrefix); } - public KeyId keyId() { - return keyId; + @Override + public int hashCode() { + return Objects.hash(secretsKeyNamePrefix); } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java new file mode 100644 index 00000000000..fa489a6b754 --- /dev/null +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/ApplicationCertificateProvider.java @@ -0,0 +1,12 @@ +package com.yahoo.vespa.hosted.controller.api.integration.certificates; + +import com.yahoo.config.provision.ApplicationId; + +/** + * Generates a certificate. + * + * @author andreer + */ +public interface ApplicationCertificateProvider { + ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId); +} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/CertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/CertificateProvider.java deleted file mode 100644 index d2462eb574f..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/CertificateProvider.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.yahoo.vespa.hosted.controller.api.integration.certificates; - -import java.security.KeyPair; -import java.security.cert.X509Certificate; -import java.util.List; - -/** - * Generates a certificate. - * - * @author andreer - */ -public interface CertificateProvider { - List requestCaSignedCertificate(KeyPair keyPair, List domains); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyId.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyId.java deleted file mode 100644 index 3ab22d4a5b7..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyId.java +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.certificates; - -/** - * Identifier for a key pair. Used for persisting/retrieving a key pair. - * - * @author mortent - * @author andreer - */ -public class KeyId { - private final String name; - private final int version; - - public KeyId(String name, int version) { - this.name = name; - this.version = version; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyPairProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyPairProvider.java deleted file mode 100644 index a872bf63343..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/KeyPairProvider.java +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.certificates; - -import com.yahoo.config.provision.ApplicationId; - -/** - * Provides a key pair. Generates and persists the key pair if not found. - * - * @author mortent - * @author andreer - */ -public interface KeyPairProvider { - VersionedKeyPair getKeyPair(ApplicationId applicationId); -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/VersionedKeyPair.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/VersionedKeyPair.java deleted file mode 100644 index c95303b9497..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/VersionedKeyPair.java +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.certificates; - -import java.security.KeyPair; - -/** - * Represents a key pair and an unique persistence identifier - * - * @author mortent - * @author andreer - */ -public class VersionedKeyPair { - private final KeyId keyId; - private final KeyPair keyPair; - - public VersionedKeyPair(KeyId keyId, KeyPair keyPair) { - this.keyId = keyId; - this.keyPair = keyPair; - } - - public KeyId keyId() { - return keyId; - } - - public KeyPair keyPair() { - return keyPair; - } -} diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 20469e6449a..ba00203ec34 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -8,6 +8,7 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.serviceview.bindings.ApplicationView; import java.io.IOException; @@ -29,7 +30,7 @@ public interface ConfigServer { } PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, - List containerEndpoints, byte[] content); + List containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content); void restart(DeploymentId deployment, Optional hostname); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index c168d09a15a..9ca73d27120 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -11,6 +11,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; import com.yahoo.vespa.hosted.controller.api.integration.organization.User; @@ -59,6 +60,7 @@ public class Application { private final Optional pemDeployKey; private final List rotations; private final Map rotationStatus; + private final Optional applicationCertificate; /** Creates an empty application */ public Application(ApplicationId id, Instant now) { @@ -66,7 +68,7 @@ public class Application { new DeploymentJobs(OptionalLong.empty(), Collections.emptyList(), Optional.empty(), false), Change.empty(), Change.empty(), Optional.empty(), Optional.empty(), OptionalInt.empty(), new ApplicationMetrics(0, 0), - Optional.empty(), Collections.emptyList(), Collections.emptyMap()); + Optional.empty(), Collections.emptyList(), Collections.emptyMap(), Optional.empty()); } /** Used from persistence layer: Do not use */ @@ -74,18 +76,19 @@ public class Application { List deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - List rotations, Map rotationStatus) { + List rotations, Map rotationStatus, + Optional applicationCertificate) { this(id, createdAt, deploymentSpec, validationOverrides, deployments.stream().collect(Collectors.toMap(Deployment::zone, Function.identity())), deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } Application(ApplicationId id, Instant createdAt, DeploymentSpec deploymentSpec, ValidationOverrides validationOverrides, Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - List rotations, Map rotationStatus) { + List rotations, Map rotationStatus, Optional applicationCertificate) { this.id = Objects.requireNonNull(id, "id cannot be null"); this.createdAt = Objects.requireNonNull(createdAt, "instant of creation cannot be null"); this.deploymentSpec = Objects.requireNonNull(deploymentSpec, "deploymentSpec cannot be null"); @@ -101,6 +104,7 @@ public class Application { this.pemDeployKey = pemDeployKey; this.rotations = List.copyOf(Objects.requireNonNull(rotations, "rotations cannot be null")); this.rotationStatus = ImmutableMap.copyOf(Objects.requireNonNull(rotationStatus, "rotationStatus cannot be null")); + this.applicationCertificate = Objects.requireNonNull(applicationCertificate, "applicationCertificate cannot be null"); } public ApplicationId id() { return id; } @@ -241,6 +245,10 @@ public class Application { .orElse(RotationStatus.unknown); } + public Optional applicationCertificate() { + return applicationCertificate; + } + @Override public boolean equals(Object o) { if (this == o) return true; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 8345aa91685..197dda8c409 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -26,6 +26,8 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; @@ -125,6 +127,8 @@ public class ApplicationController { private final Clock clock; private final BooleanFlag redirectLegacyDnsFlag; private final DeploymentTrigger deploymentTrigger; + private final BooleanFlag provisionApplicationCertificate; + private final ApplicationCertificateProvider applicationCertificateProvider; ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, RotationsConfig rotationsConfig, @@ -145,6 +149,9 @@ public class ApplicationController { this.rotationRepository = new RotationRepository(rotationsConfig, this, curator); this.deploymentTrigger = new DeploymentTrigger(controller, buildService, clock); + this.provisionApplicationCertificate = Flags.PROVISION_APPLICATION_CERTIFICATE.bindTo(controller.flagSource()); + this.applicationCertificateProvider = controller.applicationCertificateProvider(); + // Update serialization format of all applications Once.after(Duration.ofMinutes(1), () -> { Instant start = clock.instant(); @@ -287,6 +294,7 @@ public class ApplicationController { ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; Set rotationNames = new HashSet<>(); + ApplicationCertificate applicationCertificate; try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -332,6 +340,10 @@ public class ApplicationController { // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header app.rotations().stream().map(RotationId::asString).forEach(rotationNames::add); + // Get application certificate (provisions a new certificate if missing) + application = withApplicationCertificate(application); + applicationCertificate = application.get().applicationCertificate().orElse(null); + // Update application with information from application package if ( ! preferOldestVersion && ! application.get().deploymentJobs().deployedInternally() @@ -342,7 +354,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, applicationCertificate); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -409,7 +421,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of()); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ null); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -417,16 +429,16 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Set.of()); + return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ null); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set rotationNames) { + Set rotationNames, ApplicationCertificate applicationCertificate) { DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationCertificate, applicationPackage.zippedContent()); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { @@ -463,6 +475,18 @@ public class ApplicationController { return application; } + private LockedApplication withApplicationCertificate(LockedApplication application) { + ApplicationId applicationId = application.get().id(); + + // TODO: Verify that the application is deploying to a zone where certificate provisioning is enabled + boolean provisionCertificate = provisionApplicationCertificate.with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value(); + if (provisionCertificate) { + application = application.withApplicationCertificate( + Optional.of(applicationCertificateProvider.requestCaSignedCertificate(applicationId))); + } + return application; + } + private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) { Log logEntry = new Log(); logEntry.level = "WARNING"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index d87d52f2c12..ed81d08c533 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -9,12 +9,12 @@ import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; @@ -84,6 +84,7 @@ public class Controller extends AbstractComponent { private final AuditLogger auditLogger; private final FlagSource flagSource; private final NameServiceForwarder nameServiceForwarder; + private final ApplicationCertificateProvider applicationCertificateProvider; private final MavenRepository mavenRepository; /** @@ -98,12 +99,12 @@ public class Controller extends AbstractComponent { AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, - MavenRepository mavenRepository) { + MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this(curator, rotationsConfig, gitHub, zoneRegistry, configServer, metricsService, routingGenerator, chef, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, - mavenRepository); + mavenRepository, applicationCertificateProvider); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, @@ -113,7 +114,7 @@ public class Controller extends AbstractComponent { AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier hostnameSupplier, - Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository) { + Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this.hostnameSupplier = Objects.requireNonNull(hostnameSupplier, "HostnameSupplier cannot be null"); this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); @@ -126,6 +127,7 @@ public class Controller extends AbstractComponent { this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); this.nameServiceForwarder = new NameServiceForwarder(curator); + this.applicationCertificateProvider = Objects.requireNonNull(applicationCertificateProvider); this.mavenRepository = Objects.requireNonNull(mavenRepository, "MavenRepository cannot be null"); jobController = new JobController(this, runDataStore, Objects.requireNonNull(testerCloud)); @@ -304,6 +306,10 @@ public class Controller extends AbstractComponent { return auditLogger; } + public ApplicationCertificateProvider applicationCertificateProvider() { + return applicationCertificateProvider; + } + /** Returns all other roles the given tenant role implies. */ public Set impliedRoles(TenantRole role) { return Stream.concat(Roles.tenantRoles(role.tenant()).stream(), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java index 02b0afdd48f..294dc10d0bd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/LockedApplication.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.organization.IssueId; @@ -59,6 +60,7 @@ public class LockedApplication { private final Optional pemDeployKey; private final List rotations; private final Map rotationStatus; + private final Optional applicationCertificate; /** * Used to create a locked application @@ -72,7 +74,7 @@ public class LockedApplication { application.deployments(), application.deploymentJobs(), application.change(), application.outstandingChange(), application.ownershipIssueId(), application.owner(), application.majorVersion(), application.metrics(), - application.pemDeployKey(), application.assignedRotations(), application.rotationStatus()); + application.pemDeployKey(), application.assignedRotations(), application.rotationStatus(), application.applicationCertificate()); } private LockedApplication(Lock lock, ApplicationId id, Instant createdAt, @@ -80,7 +82,7 @@ public class LockedApplication { Map deployments, DeploymentJobs deploymentJobs, Change change, Change outstandingChange, Optional ownershipIssueId, Optional owner, OptionalInt majorVersion, ApplicationMetrics metrics, Optional pemDeployKey, - List rotations, Map rotationStatus) { + List rotations, Map rotationStatus, Optional applicationCertificate) { this.lock = lock; this.id = id; this.createdAt = createdAt; @@ -97,41 +99,42 @@ public class LockedApplication { this.pemDeployKey = pemDeployKey; this.rotations = rotations; this.rotationStatus = rotationStatus; + this.applicationCertificate = applicationCertificate; } /** Returns a read-only copy of this */ public Application get() { return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withBuiltInternally(boolean builtInternally) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withBuiltInternally(builtInternally), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withProjectId(OptionalLong projectId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withProjectId(projectId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withDeploymentIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.with(issueId), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withJobPause(JobType jobType, OptionalLong pausedUntil) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withPause(jobType, pausedUntil), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withJobCompletion(long projectId, JobType jobType, JobStatus.JobRun completion, @@ -139,14 +142,14 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withCompletion(projectId, jobType, completion, jobError), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, rotations, rotationStatus); + pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withJobTriggering(JobType jobType, JobStatus.JobRun job) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.withTriggering(jobType, job), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication withNewDeployment(ZoneId zone, ApplicationVersion applicationVersion, Version version, @@ -197,45 +200,45 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs.without(jobType), change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication with(DeploymentSpec deploymentSpec) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } public LockedApplication with(ValidationOverrides validationOverrides) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withChange(Change change) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withOutstandingChange(Change outstandingChange) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withOwnershipIssueId(IssueId issueId) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, Optional.ofNullable(issueId), owner, - majorVersion, metrics, pemDeployKey, rotations, rotationStatus); + majorVersion, metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withOwner(User owner) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, Optional.ofNullable(owner), majorVersion, metrics, pemDeployKey, - rotations, rotationStatus); + rotations, rotationStatus, applicationCertificate); } /** Set a major version for this, or set to null to remove any major version override */ @@ -243,33 +246,40 @@ public class LockedApplication { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion == null ? OptionalInt.empty() : OptionalInt.of(majorVersion), - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication with(MetricsService.ApplicationMetrics metrics) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } public LockedApplication withPemDeployKey(String pemDeployKey) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus); + metrics, Optional.ofNullable(pemDeployKey), rotations, rotationStatus, applicationCertificate); } public LockedApplication with(List assignedRotations) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, assignedRotations, rotationStatus); + metrics, pemDeployKey, assignedRotations, rotationStatus, applicationCertificate); } public LockedApplication withRotationStatus(Map rotationStatus) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } + public LockedApplication withApplicationCertificate(Optional applicationCertificate) { + return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, + deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); + } + + /** Don't expose non-leaf sub-objects. */ private LockedApplication with(Deployment deployment) { Map deployments = new LinkedHashMap<>(this.deployments); @@ -280,7 +290,7 @@ public class LockedApplication { private LockedApplication with(Map deployments) { return new LockedApplication(lock, id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, change, outstandingChange, ownershipIssueId, owner, majorVersion, - metrics, pemDeployKey, rotations, rotationStatus); + metrics, pemDeployKey, rotations, rotationStatus, applicationCertificate); } @Override diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 271221e15b2..6ecf60e7404 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -15,6 +15,7 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService.ApplicationMetrics; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; @@ -87,6 +88,7 @@ public class ApplicationSerializer { private final String rotationsField = "endpoints"; private final String deprecatedRotationField = "rotation"; private final String rotationStatusField = "rotationStatus"; + private final String applicationCertificateField = "applicationCertificate"; // Deployment fields private final String zoneField = "zone"; @@ -181,6 +183,7 @@ public class ApplicationSerializer { rotationsToSlime(application.assignedRotations(), root, rotationsField); assignedRotationsToSlime(application.assignedRotations(), root, assignedRotationsField); toSlime(application.rotationStatus(), root.setArray(rotationStatusField)); + application.applicationCertificate().ifPresent(cert -> root.setString(applicationCertificateField, cert.secretsKeyNamePrefix())); return slime; } @@ -363,10 +366,11 @@ public class ApplicationSerializer { Optional pemDeployKey = optionalString(root.field(pemDeployKeyField)); List assignedRotations = assignedRotationsFromSlime(deploymentSpec, root); Map rotationStatus = rotationStatusFromSlime(root.field(rotationStatusField)); + Optional applicationCertificate = optionalString(root.field(applicationCertificateField)).map(ApplicationCertificate::new); return new Application(id, createdAt, deploymentSpec, validationOverrides, deployments, deploymentJobs, deploying, outstandingChange, ownershipIssueId, owner, majorVersion, metrics, - pemDeployKey, assignedRotations, rotationStatus); + pemDeployKey, assignedRotations, rotationStatus, applicationCertificate); } private List deploymentsFromSlime(Inspector array) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index 5748ad4f55c..dbf983a5bab 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -5,6 +5,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Slime; import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -35,11 +36,11 @@ import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMavenRepository; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockRunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockTesterCloud; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzClientFactoryMock; import com.yahoo.vespa.hosted.controller.athenz.mock.AthenzDbMock; +import com.yahoo.vespa.hosted.controller.integration.ApplicationCertificateMock; import com.yahoo.vespa.hosted.controller.integration.ApplicationStoreMock; import com.yahoo.vespa.hosted.controller.integration.ArtifactRepositoryMock; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; @@ -353,7 +354,8 @@ public final class ControllerTester { () -> "test-controller", new MockMailer(), new InMemoryFlagSource(), - new MockMavenRepository()); + new MockMavenRepository(), + new ApplicationCertificateMock()); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationCertificateMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationCertificateMock.java new file mode 100644 index 00000000000..3246a260217 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ApplicationCertificateMock.java @@ -0,0 +1,14 @@ +// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.integration; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; + +public class ApplicationCertificateMock implements ApplicationCertificateProvider { + + @Override + public ApplicationCertificate requestCaSignedCertificate(ApplicationId applicationId) { + return new ApplicationCertificate(String.format("vespa.tls.%s.%s", applicationId.tenant(),applicationId.application())); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index d4df9c20ead..cdbc45c4d8f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -15,6 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -225,7 +226,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, - List containerEndpoints, byte[] content) { + List containerEndpoints, ApplicationCertificate applicationCertificate, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 67b6b1ac61c..347fe6064df 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -10,6 +10,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; @@ -119,7 +120,8 @@ public class ApplicationSerializerTest { new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"))), - rotationStatus); + rotationStatus, + Optional.of(new ApplicationCertificate("vespa.certificate"))); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -157,6 +159,8 @@ public class ApplicationSerializerTest { assertEquals(original.rotations(), serialized.rotations()); assertEquals(original.rotationStatus(), serialized.rotationStatus()); + assertEquals(original.applicationCertificate(), serialized.applicationCertificate()); + // Test cluster utilization assertEquals(0, serialized.deployments().get(zone1).clusterUtils().size()); assertEquals(3, serialized.deployments().get(zone2).clusterUtils().size()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 6f612005524..53476a2e42f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -92,6 +92,7 @@ public class ControllerContainerTest { " \n" + " \n" + " \n" + + " \n" + " \n" + " \n" + " http://*/deployment/v1/*\n" + diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 1b7ed1fb21e..8ae98247121 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -151,6 +151,11 @@ public class Flags { "Configserver RPC authorizer. Allowed values: ['disable', 'log-only', 'enforce']", "Takes effect on restart of configserver"); + public static final UnboundBooleanFlag PROVISION_APPLICATION_CERTIFICATE = defineFeatureFlag( + "provision-application-certificate", false, + "Privision certificate from CA and include reference in deployment", + "Takes effect on deployment through controller", + APPLICATION_ID); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, -- cgit v1.2.3 From e6b4f5de4f2fb60aadf518a55b80ca9db8d665cd Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Wed, 26 Jun 2019 13:14:24 +0200 Subject: Add support for EndpointId in EndpointBuilder --- .../hosted/controller/application/Endpoint.java | 20 ++++++++-- .../controller/application/EndpointTest.java | 44 ++++++++++++++++++++++ 2 files changed, 60 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 5026ca75a83..5dccd5c8120 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -217,6 +217,7 @@ public class Endpoint { private ZoneId zone; private ClusterSpec.Id cluster; private RotationName rotation; + private EndpointId endpointId; private Port port; private boolean legacy = false; private boolean directRouting = false; @@ -227,8 +228,8 @@ public class Endpoint { /** Sets the cluster and zone target of this */ public EndpointBuilder target(ClusterSpec.Id cluster, ZoneId zone) { - if (rotation != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if (rotation != null || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.cluster = cluster; this.zone = zone; @@ -237,13 +238,22 @@ public class Endpoint { /** Sets the rotation target of this */ public EndpointBuilder target(RotationName rotation) { - if (cluster != null && zone != null) { - throw new IllegalArgumentException("Cannot set both cluster and rotation target"); + if ((cluster != null && zone != null) || endpointId != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); } this.rotation = rotation; return this; } + /** Sets the endpoint ID as defines in deployments.xml */ + public EndpointBuilder named(EndpointId endpointId) { + if (rotation != null || cluster != null || zone != null) { + throw new IllegalArgumentException("Cannot set multiple target types"); + } + this.endpointId = endpointId; + return this; + } + /** Sets the port of this */ public EndpointBuilder on(Port port) { this.port = port; @@ -269,6 +279,8 @@ public class Endpoint { name = cluster.value(); } else if (rotation != null) { name = rotation.value(); + } else if (endpointId != null) { + name = endpointId.id(); } else { throw new IllegalArgumentException("Must set either cluster or rotation target"); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index 16b875c1892..f5047a82e2f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -65,6 +65,50 @@ public class EndpointTest { tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); } + @Test + public void test_global_endpoints_with_endpoint_id() { + final var endpointId = EndpointId.default_(); + + Map tests = Map.of( + // Legacy endpoint + "http://a1.t1.global.vespa.yahooapis.com:4080/", + Endpoint.of(app1).named(endpointId).on(Port.plain(4080)).legacy().in(SystemName.main), + + // Legacy endpoint with TLS + "https://a1--t1.global.vespa.yahooapis.com:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).legacy().in(SystemName.main), + + // Main endpoint + "https://a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.main), + + // Main endpoint in CD + "https://cd--a1--t1.global.vespa.oath.cloud:4443/", + Endpoint.of(app1).named(endpointId).on(Port.tls(4443)).in(SystemName.cd), + + // Main endpoint with direct routing and default TLS port + "https://a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint with custom rotation name + "https://r1.a1.t1.global.vespa.oath.cloud/", + Endpoint.of(app1).named(EndpointId.of("r1")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance in default rotation + "https://a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).named(endpointId).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint for custom instance with custom rotation name + "https://r2.a2.t2.global.vespa.oath.cloud/", + Endpoint.of(app2).named(EndpointId.of("r2")).on(Port.tls()).directRouting().in(SystemName.main), + + // Main endpoint in public system + "https://a1.t1.global.public.vespa.oath.cloud/", + Endpoint.of(app1).named(endpointId).on(Port.tls()).directRouting().in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + } + @Test public void test_zone_endpoints() { ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); // Always default for non-direct routing -- cgit v1.2.3 From c48905631b34984bbeb9832de607932564194057 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Thu, 27 Jun 2019 17:43:04 +0200 Subject: Add support for endpoints in APB --- .../deployment/ApplicationPackageBuilder.java | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index 83b95ccc8b0..6635547e9be 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -17,6 +17,9 @@ import java.time.Duration; import java.time.Instant; import java.util.Arrays; import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; import java.util.OptionalInt; import java.util.StringJoiner; import java.util.zip.ZipEntry; @@ -35,6 +38,7 @@ public class ApplicationPackageBuilder { private final StringJoiner notifications = new StringJoiner("/>\n \n \n\n").setEmptyValue(""); + private final StringBuilder endpointsBody = new StringBuilder(); private OptionalInt majorVersion = OptionalInt.empty(); private String upgradePolicy = null; @@ -63,6 +67,18 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder endpoint(String endpointId, String containerId, String... regions) { + endpointsBody.append(" \n"); + for (var region : regions) { + endpointsBody.append(" ").append(region).append("\n"); + } + endpointsBody.append(" \n"); + return this; + } + public ApplicationPackageBuilder region(RegionName regionName) { return region(regionName.value()); } @@ -157,7 +173,11 @@ public class ApplicationPackageBuilder { xml.append(environmentBody); xml.append(" \n"); + xml.append(">\n"); + xml.append(" \n"); + xml.append(endpointsBody); + xml.append(" \n"); + xml.append(""); return xml.toString().getBytes(StandardCharsets.UTF_8); } -- cgit v1.2.3 From 80fc53aca53aaa1722473db6f311a4c864a8bd40 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 09:51:18 +0200 Subject: Remove "support" for duplicate endpoint IDs in ApplicationSerializer --- .../controller/persistence/ApplicationSerializer.java | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index 271221e15b2..23c76b95950 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -48,6 +49,7 @@ import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; import java.util.TreeMap; +import java.util.stream.Collectors; /** * Serializes {@link Application} to/from slime. @@ -547,23 +549,25 @@ public class ApplicationSerializer { } private List assignedRotationsFromSlime(DeploymentSpec deploymentSpec, Inspector root) { - final var assignedRotations = new LinkedHashSet(); + final var assignedRotations = new LinkedHashMap(); // Add the legacy rotation field to the set - this needs to be first // TODO: Remove when we retire the rotations field final var legacyRotation = legacyRotationFromSlime(root.field(deprecatedRotationField)); if (legacyRotation.isPresent() && deploymentSpec.globalServiceId().isPresent()) { final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); - assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get())); + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), legacyRotation.get(), regions)); } // Now add the same entries from "stupid" list of rotations // TODO: Remove when we retire the rotations field final var rotations = rotationListFromSlime(root.field(rotationsField)); for (var rotation : rotations) { + final var regions = deploymentSpec.zones().stream().flatMap(zone -> zone.region().stream()).collect(Collectors.toSet()); if (deploymentSpec.globalServiceId().isPresent()) { final var clusterId = new ClusterSpec.Id(deploymentSpec.globalServiceId().get()); - assignedRotations.add(new AssignedRotation(clusterId, EndpointId.default_(), rotation)); + assignedRotations.putIfAbsent(EndpointId.default_(), new AssignedRotation(clusterId, EndpointId.default_(), rotation, regions)); } } @@ -572,10 +576,14 @@ public class ApplicationSerializer { final var clusterId = new ClusterSpec.Id(inspector.field(assignedRotationClusterField).asString()); final var endpointId = EndpointId.of(inspector.field(assignedRotationEndpointField).asString()); final var rotationId = new RotationId(inspector.field(assignedRotationRotationField).asString()); - assignedRotations.add(new AssignedRotation(clusterId, endpointId, rotationId)); + final var regions = deploymentSpec.endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId.id())) + .flatMap(endpoint -> endpoint.regions().stream()) + .collect(Collectors.toSet()); + assignedRotations.putIfAbsent(endpointId, new AssignedRotation(clusterId, endpointId, rotationId, regions)); }); - return List.copyOf(assignedRotations); + return List.copyOf(assignedRotations.values()); } private List rotationListFromSlime(Inspector field) { -- cgit v1.2.3 From 31234b9917c32813c02cc723900677ea0f140d8e Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 09:52:08 +0200 Subject: Put information on assigned regions in AssignedRotation class --- .../hosted/controller/application/AssignedRotation.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index e1ed278a79e..4906d60fe21 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -1,12 +1,14 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.rotation.RotationId; import java.util.Objects; +import java.util.Set; /** - * Contains the tuple of [clusterId, endpointId, rotationId], to keep track + * Contains the tuple of [clusterId, endpointId, rotationId, regions[]], to keep track * of which services have assigned which rotations under which name. * * @author ogronnesby @@ -15,16 +17,19 @@ public class AssignedRotation { private final ClusterSpec.Id clusterId; private final EndpointId endpointId; private final RotationId rotationId; + private final Set regions; - public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId) { + public AssignedRotation(ClusterSpec.Id clusterId, EndpointId endpointId, RotationId rotationId, Set regions) { this.clusterId = requireNonEmpty(clusterId, clusterId.value(), "clusterId"); this.endpointId = Objects.requireNonNull(endpointId); this.rotationId = Objects.requireNonNull(rotationId); + this.regions = Set.copyOf(Objects.requireNonNull(regions)); } public ClusterSpec.Id clusterId() { return clusterId; } public EndpointId endpointId() { return endpointId; } public RotationId rotationId() { return rotationId; } + public Set regions() { return regions; } @Override public String toString() { @@ -32,6 +37,7 @@ public class AssignedRotation { "clusterId=" + clusterId + ", endpointId='" + endpointId + '\'' + ", rotationId=" + rotationId + + ", regions=" + regions + '}'; } @@ -42,12 +48,13 @@ public class AssignedRotation { AssignedRotation that = (AssignedRotation) o; return clusterId.equals(that.clusterId) && endpointId.equals(that.endpointId) && - rotationId.equals(that.rotationId); + rotationId.equals(that.rotationId) && + regions.equals(that.regions); } @Override public int hashCode() { - return Objects.hash(clusterId, endpointId, rotationId); + return Objects.hash(clusterId, endpointId, rotationId, regions); } private static T requireNonEmpty(T object, String value, String field) { -- cgit v1.2.3 From 44e40370760befb611100e0afacc701b4c027ebd Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 09:55:30 +0200 Subject: Add methods to get rotations based on ID, and to assign multiple rotations per app --- .../controller/rotation/RotationRepository.java | 64 ++++++++++++++++++++++ 1 file changed, 64 insertions(+) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index d2b16721503..7165d134d59 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -1,18 +1,26 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.rotation; +import com.yahoo.collections.Pair; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import java.util.ArrayList; import java.util.Collections; import java.util.Comparator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; +import java.util.function.Function; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -50,6 +58,11 @@ public class RotationRepository { return application.rotations().stream().map(allRotations::get).findFirst(); } + /** Get rotation for the given rotationId */ + public Optional getRotation(RotationId rotationId) { + return Optional.of(allRotations.get(rotationId)); + } + /** * Returns a rotation for the given application * @@ -75,6 +88,57 @@ public class RotationRepository { return findAvailableRotation(application, lock); } + public List getOrAssignRotations(Application application, RotationLock lock) { + if (application.deploymentSpec().globalServiceId().isPresent() && ! application.deploymentSpec().endpoints().isEmpty()) { + throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); + } + + // Support the older case of setting global-service-id + if (application.deploymentSpec().globalServiceId().isPresent()) { + final var regions = application.deploymentSpec().zones().stream() + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var rotation = getOrAssignRotation(application, lock); + + return List.of( + new AssignedRotation( + new ClusterSpec.Id(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ) + ); + } + + final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + final var assignments = application.assignedRotations().stream() + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + Function.identity(), + (a, b) -> { throw new IllegalStateException("Duplicate entries: " + a + ", " + b); }, + LinkedHashMap::new + ) + ); + + application.deploymentSpec().endpoints().stream() + .filter(endpoint -> ! assignments.containsKey(new EndpointId(endpoint.endpointId()))) + .map(endpoint -> { + return new AssignedRotation( + new ClusterSpec.Id(endpoint.containerId()), + EndpointId.of(endpoint.endpointId()), + availableRotations.remove(0).id(), + endpoint.regions() + ); + }) + .forEach(assignment -> { + assignments.put(assignment.endpointId(), assignment); + }); + + return List.copyOf(assignments.values()); + } + /** * Returns all unassigned rotations * @param lock Lock which must be acquired by the caller -- cgit v1.2.3 From c026958c9e37145a2284f0b498331feacb31719c Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 09:56:21 +0200 Subject: Take endpoints into account when generating EndpointList --- .../com/yahoo/vespa/hosted/controller/Application.java | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index c168d09a15a..b5bf5a7cc98 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -6,6 +6,7 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; @@ -19,6 +20,8 @@ import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; import com.yahoo.vespa.hosted.controller.rotation.RotationId; @@ -214,12 +217,18 @@ public class Application { return rotations; } + /** Returns the default global endpoints for this in given system - for a given endpoint ID */ + public EndpointList endpointsIn(SystemName system, EndpointId endpointId) { + if (rotations.isEmpty()) return EndpointList.EMPTY; + return EndpointList.defaultGlobal(id, endpointId, system); + } + /** Returns the default global endpoints for this in given system */ public EndpointList endpointsIn(SystemName system) { - // TODO: Do we need to change something here? .defaultGlobalId seems like it is - // TODO: making some assumptions on naming. if (rotations.isEmpty()) return EndpointList.EMPTY; - return EndpointList.defaultGlobal(id, system); + final var endpointStream = rotations.stream() + .flatMap(rotation -> EndpointList.defaultGlobal(id, rotation.endpointId(), system).asList().stream()); + return EndpointList.of(endpointStream); } public Optional pemDeployKey() { return pemDeployKey; } -- cgit v1.2.3 From 5c06c9a27f509beea23ba7815dd8f054a023287d Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 09:58:27 +0200 Subject: Use EndpointId to generate Endpoints in EndpointList --- .../vespa/hosted/controller/application/EndpointList.java | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 0c04a1f099c..216b6c1eab9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -67,16 +67,14 @@ public class EndpointList { } /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ - public static EndpointList defaultGlobal(ApplicationId application, SystemName system) { - // Rotation name is always default in the routing layer - RotationName rotation = RotationName.from("default"); + public static EndpointList defaultGlobal(ApplicationId application, EndpointId endpointId, SystemName system) { switch (system) { case cd: case main: return new EndpointList(List.of( - Endpoint.of(application).target(rotation).on(Port.plain(4080)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).legacy().in(system), - Endpoint.of(application).target(rotation).on(Port.tls(4443)).in(system) + Endpoint.of(application).named(endpointId).on(Port.plain(4080)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).legacy().in(system), + Endpoint.of(application).named(endpointId).on(Port.tls(4443)).in(system) )); } return EMPTY; -- cgit v1.2.3 From 799bfebc3799cb875e1e54eef8fd5cd24a348258 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 09:59:47 +0200 Subject: Assign multiple rotations per application during deploy Use updated logic to generate endpoint names, use the new interface to send ContainerEndpoints to the configserver, and use new method to assign multiple rotations per application. --- .../hosted/controller/ApplicationController.java | 76 ++++++++++++++-------- 1 file changed, 50 insertions(+), 26 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 8345aa91685..61a4e1e8656 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -28,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NotFoundException; @@ -85,6 +86,7 @@ import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; @@ -95,6 +97,7 @@ import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.active; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State.reserved; @@ -286,7 +289,7 @@ public class ApplicationController { Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; - Set rotationNames = new HashSet<>(); + Set endpoints = new LinkedHashSet<>(); try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(require(applicationId), lock); @@ -328,9 +331,18 @@ public class ApplicationController { application = withRotation(application, zone); Application app = application.get(); // Include global DNS names - app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(rotationNames::add); - // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header - app.rotations().stream().map(RotationId::asString).forEach(rotationNames::add); + app.assignedRotations().stream() + .filter(assignedRotation -> assignedRotation.regions().contains(zone.region())) + .map(assignedRotation -> { + return new ContainerEndpoint( + assignedRotation.clusterId().value(), + Stream.concat( + app.endpointsIn(controller.system(), assignedRotation.endpointId()).legacy(false).asList().stream().map(Endpoint::dnsName), + app.rotations().stream().map(RotationId::asString) + ).collect(Collectors.toList()) + ); + }) + .forEach(endpoints::add); // Update application with information from application package if ( ! preferOldestVersion @@ -342,7 +354,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, endpoints); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -422,11 +434,11 @@ public class ApplicationController { private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set rotationNames) { + Set endpoints) { DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, Set.of(), endpoints, applicationPackage.zippedContent()); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { @@ -438,29 +450,35 @@ public class ApplicationController { /** Makes sure the application has a global rotation, if eligible. */ private LockedApplication withRotation(LockedApplication application, ZoneId zone) { - if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) { + if (zone.environment() == Environment.prod && applicationNeedsRotations(application.get().deploymentSpec())) { try (RotationLock rotationLock = rotationRepository.lock()) { - Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock); - application = application.with(List.of(new AssignedRotation(new ClusterSpec.Id(application.get().deploymentSpec().globalServiceId().get()), EndpointId.default_(), rotation.id()))); + final var rotations = rotationRepository.getOrAssignRotations(application.get(), rotationLock); + application = application.with(rotations); store(application); // store assigned rotation even if deployment fails + registerAssignedRotationCnames(application.get()); + } + } + return application; + } + + private boolean applicationNeedsRotations(DeploymentSpec spec) { + return spec.globalServiceId().isPresent() || !spec.endpoints().isEmpty(); + } - boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) - .value(); + private void registerAssignedRotationCnames(Application application) { + application.assignedRotations().forEach(assignedRotation -> { + final var endpoints = application + .endpointsIn(controller.system(), assignedRotation.endpointId()) + .scope(Endpoint.Scope.global); - EndpointList globalEndpoints = application.get() - .endpointsIn(controller.system()) - .scope(Endpoint.Scope.global); - globalEndpoints.main().ifPresent(mainEndpoint -> { + final var maybeRotation = rotationRepository.getRotation(assignedRotation.rotationId()); + + maybeRotation.ifPresent(rotation -> { + endpoints.main().ifPresent(mainEndpoint -> { registerCname(mainEndpoint.dnsName(), rotation.name()); - if (redirectLegacyDns) { - globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); - } else { - globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); - } }); - } - } - return application; + }); + }); } private ActivateResult unexpectedDeployment(ApplicationId application, ZoneId zone) { @@ -607,8 +625,14 @@ public class ApplicationController { applicationStore.removeAll(id); applicationStore.removeAll(TesterId.of(id)); - EndpointList endpoints = application.get().endpointsIn(controller.system()); - endpoints.asList().stream().map(Endpoint::dnsName).forEach(name -> controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal)); + application.get().assignedRotations().forEach(assignedRotation -> { + final var endpoints = application.get().endpointsIn(controller.system(), assignedRotation.endpointId()); + endpoints.asList().stream() + .map(Endpoint::dnsName) + .forEach(name -> { + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(name), Priority.normal); + }); + }); log.info("Deleted " + application); })); -- cgit v1.2.3 From 17a6634ad5d229c00d670b2f10e9cce8fb3cde04 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 10:00:57 +0200 Subject: Change signature from List to Set so we don't respect ordering too much --- .../hosted/controller/api/integration/configserver/ConfigServer.java | 2 +- .../yahoo/vespa/hosted/controller/integration/ConfigServerMock.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java index 20469e6449a..38e76b83fca 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServer.java @@ -29,7 +29,7 @@ public interface ConfigServer { } PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, - List containerEndpoints, byte[] content); + Set containerEndpoints, byte[] content); void restart(DeploymentId deployment, Optional hostname); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index d4df9c20ead..946cb1c67c0 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -225,7 +225,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer @Override public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set rotationNames, - List containerEndpoints, byte[] content) { + Set containerEndpoints, byte[] content) { lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null); if (prepareException != null) { RuntimeException prepareException = this.prepareException; @@ -237,7 +237,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) provision(deployment.zoneId(), deployment.applicationId()); - this.rotationNames.put(deployment, Set.copyOf(rotationNames)); + this.rotationNames.put(deployment, containerEndpoints.stream().flatMap(e -> e.names().stream()).collect(Collectors.toSet())); return () -> { Application application = applications.get(deployment.applicationId()); -- cgit v1.2.3 From c81bd8b7310f618985db8c4106521d79eb77ea3f Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 28 Jun 2019 10:16:02 +0200 Subject: Update tests to take into account rotation changes - We no longer generate legacy endpoint names - We can have multiple global endpoints per application - We only generate one global endpoint per endpoint id --- .../vespa/hosted/controller/ControllerTest.java | 105 +++++++++------------ .../maintenance/MetricsReporterTest.java | 2 +- .../persistence/ApplicationSerializerTest.java | 22 +++-- 3 files changed, 58 insertions(+), 71 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index de31f1f67f9..3a905f8c6d5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -290,59 +290,57 @@ public class ControllerTest { for (Deployment deployment : deployments) { assertEquals("Rotation names are passed to config server in " + deployment.zone(), Set.of("rotation-id-01", - "app1--tenant1.global.vespa.oath.cloud", - "app1.tenant1.global.vespa.yahooapis.com", - "app1--tenant1.global.vespa.yahooapis.com"), + "app1--tenant1.global.vespa.oath.cloud"), tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); } tester.flushDnsRequests(); - assertEquals(3, tester.controllerTester().nameService().records().size()); - Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + var record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); } @Test - public void testRedirectLegacyDnsNames() { // TODO: Remove together with Flags.REDIRECT_LEGACY_DNS_NAMES + public void testDnsAliasRegistrationWithEndpoints() { Application application = tester.createApplication("app1", "tenant1", 1, 1L); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) - .globalServiceId("foo") + .endpoint("foobar", "qrs", "us-west-1", "us-central-1") + .endpoint("default", "qrs", "us-west-1", "us-central-1") .region("us-west-1") - .region("us-central-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once .build(); - ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.REDIRECT_LEGACY_DNS_NAMES.id(), true); - tester.deployCompletely(application, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); + Collection deployments = tester.application(application.id()).deployments().values(); + assertFalse(deployments.isEmpty()); + for (Deployment deployment : deployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of( + "rotation-id-01", + "rotation-id-02", + "app1--tenant1.global.vespa.oath.cloud", + "foobar--app1--tenant1.global.vespa.oath.cloud" + ), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); + } + tester.flushDnsRequests(); - Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("app1--tenant1.global.vespa.oath.cloud.", record.get().data().asString()); + assertEquals(2, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record1.isPresent()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("app1--tenant1.global.vespa.oath.cloud.", record.get().data().asString()); + var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record2.isPresent()); + assertEquals("foobar--app1--tenant1.global.vespa.oath.cloud", record2.get().name().asString()); + assertEquals("rotation-fqdn-01.", record2.get().data().asString()); } @Test @@ -358,16 +356,11 @@ public class ControllerTest { .build(); tester.deployCompletely(app1, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); + Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record.isPresent()); - assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); // Application is deleted and rotation is unassigned @@ -408,22 +401,12 @@ public class ControllerTest { .region("us-central-1") .build(); tester.deployCompletely(app2, applicationPackage); - assertEquals(3, tester.controllerTester().nameService().records().size()); - - Optional record = tester.controllerTester().findCname("app2--tenant2.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app2--tenant2.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); + assertEquals(1, tester.controllerTester().nameService().records().size()); - record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + var record = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); assertTrue(record.isPresent()); assertEquals("app2--tenant2.global.vespa.oath.cloud", record.get().name().asString()); assertEquals("rotation-fqdn-01.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app2.tenant2.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("app2.tenant2.global.vespa.yahooapis.com", record.get().name().asString()); - assertEquals("rotation-fqdn-01.", record.get().data().asString()); } // Application 1 is recreated, deployed and assigned a new rotation @@ -441,19 +424,15 @@ public class ControllerTest { assertEquals("rotation-id-02", app1.rotations().get(0).asString()); // DNS records are created for the newly assigned rotation - assertEquals(6, tester.controllerTester().nameService().records().size()); + assertEquals(2, tester.controllerTester().nameService().records().size()); - Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); + var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record1.isPresent()); + assertEquals("rotation-fqdn-02.", record1.get().data().asString()); - record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); - - record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); - assertTrue(record.isPresent()); - assertEquals("rotation-fqdn-02.", record.get().data().asString()); + var record2 = tester.controllerTester().findCname("app2--tenant2.global.vespa.oath.cloud"); + assertTrue(record2.isPresent()); + assertEquals("rotation-fqdn-01.", record2.get().data().asString()); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 58f35c0ac05..b994c55ce55 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -253,7 +253,7 @@ public class MetricsReporterTest { tester.deployCompletely(application, applicationPackage); reporter.maintain(); - assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); + assertEquals("Deployment queues name services requests", 2, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); tester.flushDnsRequests(); reporter.maintain(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 67b6b1ac61c..f9725715ce5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; @@ -45,6 +46,7 @@ import java.util.Optional; import java.util.OptionalDouble; import java.util.OptionalInt; import java.util.OptionalLong; +import java.util.Set; import java.util.TreeMap; import static com.yahoo.config.provision.SystemName.main; @@ -118,7 +120,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), - List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"))), + List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"), Set.of())), rotationStatus); Application serialized = applicationSerializer.fromSlime(applicationSerializer.toSlime(original)); @@ -254,6 +256,11 @@ public class ApplicationSerializerTest { final var applicationJson = Files.readAllBytes(testData.resolve("complete-application.json")); final var slime = SlimeUtils.jsonToSlime(applicationJson); + final var regions = Set.of( + RegionName.from("us-east-3"), + RegionName.from("us-west-1") + ); + // Add the necessary fields to the Slime representation of the application final var cursor = slime.get(); cursor.setString("rotation", "single-rotation"); @@ -271,11 +278,11 @@ public class ApplicationSerializerTest { // Parse and test the output from parsing contains both legacy rotation and multiple rotations final var application = applicationSerializer.fromSlime(slime); + // Since only one AssignedEndpoint can be "default", we make sure that we are ignoring the + // multiple-rotation entries as the globalServiceId will override them assertEquals( List.of( new RotationId("single-rotation"), - new RotationId("multiple-rotation-1"), - new RotationId("multiple-rotation-2"), new RotationId("assigned-rotation") ), application.rotations() @@ -285,12 +292,13 @@ public class ApplicationSerializerTest { Optional.of(new RotationId("single-rotation")), application.legacyRotation() ); + // The same goes here for AssignedRotations with "default" EndpointId as in the .rotations() test above. + // Note that we are only using Set.of() on "assigned-rotation" because in this test we do not have access + // to a deployment.xml that describes the zones a rotation should map to. assertEquals( List.of( - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation")), - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-1")), - new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("multiple-rotation-2")), - new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation")) + new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.of("default"), new RotationId("single-rotation"), regions), + new AssignedRotation(new ClusterSpec.Id("foobar"), EndpointId.of("nice-endpoint"), new RotationId("assigned-rotation"), Set.of()) ), application.assignedRotations() ); -- cgit v1.2.3 From ad2a8af7050663356463a884c3714a6bd2b93afb Mon Sep 17 00:00:00 2001 From: andreer Date: Fri, 28 Jun 2019 12:31:39 +0200 Subject: retry when certificate not ready --- .../api/integration/configserver/ConfigServerException.java | 3 ++- .../yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java index a2d199a38a8..20599e92aa9 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ConfigServerException.java @@ -37,7 +37,8 @@ public class ConfigServerException extends RuntimeException { OUT_OF_CAPACITY, REQUEST_TIMEOUT, UNKNOWN_VESPA_VERSION, - PARENT_HOST_NOT_READY + PARENT_HOST_NOT_READY, + CERTIFICATE_NOT_READY } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index bf2460284ab..ce0e7c0dbab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -61,6 +61,7 @@ import static com.yahoo.log.LogLevel.DEBUG; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.ACTIVATION_CONFLICT; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.APPLICATION_LOCK_FAILURE; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.BAD_REQUEST; +import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.CERTIFICATE_NOT_READY; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.INVALID_APPLICATION_PACKAGE; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.OUT_OF_CAPACITY; import static com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException.ErrorCode.PARENT_HOST_NOT_READY; @@ -231,7 +232,8 @@ public class InternalStepRunner implements StepRunner { if ( e.getErrorCode() == OUT_OF_CAPACITY && type.isTest() || e.getErrorCode() == ACTIVATION_CONFLICT || e.getErrorCode() == APPLICATION_LOCK_FAILURE - || e.getErrorCode() == PARENT_HOST_NOT_READY) { + || e.getErrorCode() == PARENT_HOST_NOT_READY + || e.getErrorCode() == CERTIFICATE_NOT_READY) { logger.log("Will retry, because of '" + e.getErrorCode() + "' deploying:\n" + e.getMessage()); return Optional.empty(); } -- cgit v1.2.3 From 7c26f5753befec8cf28adc4a86b9e67308f657c5 Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Tue, 2 Jul 2019 11:06:58 +0200 Subject: Some methods in SslContextFactory base class are deprecated Use SslContextFactory.Server instead. --- .../instanceproviderservice/ConfigserverSslContextFactoryProvider.java | 2 +- .../hosted/controller/tls/ControllerSslContextFactoryProvider.java | 2 +- .../yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java | 2 +- .../main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java | 2 +- .../yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) (limited to 'controller-server') diff --git a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java index 2bda2eb3627..0eeeae457b6 100644 --- a/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java +++ b/athenz-identity-provider-service/src/main/java/com/yahoo/vespa/hosted/athenz/instanceproviderservice/ConfigserverSslContextFactoryProvider.java @@ -110,7 +110,7 @@ public class ConfigserverSslContextFactoryProvider extends AbstractComponent imp // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints) - SslContextFactory factory = new SslContextFactory(); + SslContextFactory.Server factory = new SslContextFactory.Server(); factory.setWantClientAuth(true); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index 84d43314412..d50d141d625 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -58,7 +58,7 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple private SslContextFactory createSslContextFactory(int port) { // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints). - SslContextFactory factory = new SslContextFactory(); + SslContextFactory.Server factory = new SslContextFactory.Server(); if (port != 443) { factory.setWantClientAuth(true); } diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java index 2021105fc52..22168352fbe 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/ConfiguredSslContextFactoryProvider.java @@ -37,7 +37,7 @@ public class ConfiguredSslContextFactoryProvider implements SslContextFactoryPro public SslContextFactory getInstance(String containerId, int port) { ConnectorConfig.Ssl sslConfig = connectorConfig.ssl(); if (!sslConfig.enabled()) throw new IllegalStateException(); - SslContextFactory factory = new JDiscSslContextFactory(); + SslContextFactory.Server factory = new JDiscSslContextFactory(); switch (sslConfig.clientAuth()) { case NEED_AUTH: diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java index 90d67996402..4d3bb4a280a 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/JDiscSslContextFactory.java @@ -13,7 +13,7 @@ import java.util.Objects; * * @author bjorncs */ -class JDiscSslContextFactory extends SslContextFactory { +class JDiscSslContextFactory extends SslContextFactory.Server { private String trustStorePassword; diff --git a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java index d7d28e7d242..a5652042f9e 100644 --- a/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java +++ b/jdisc_http_service/src/main/java/com/yahoo/jdisc/http/ssl/impl/TlsContextManagedSslContextFactory.java @@ -13,7 +13,7 @@ import java.net.InetSocketAddress; * * @author bjorncs */ -class TlsContextManagedSslContextFactory extends SslContextFactory { +class TlsContextManagedSslContextFactory extends SslContextFactory.Server { private final TlsContext tlsContext; -- cgit v1.2.3 From 503fdaddd2ee6dc4c829786d8402399cef0d2d75 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 2 Jul 2019 11:24:18 +0200 Subject: Add convenience method to create AssignedRotations --- .../hosted/controller/application/AssignedRotation.java | 12 ++++++++++++ .../controller/persistence/ApplicationSerializerTest.java | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 4906d60fe21..ec13066d069 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -4,8 +4,10 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import java.util.Collection; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; /** * Contains the tuple of [clusterId, endpointId, rotationId, regions[]], to keep track @@ -65,4 +67,14 @@ public class AssignedRotation { } return object; } + + /** Convenience method intended for tests */ + public static AssignedRotation fromStrings(String clusterId, String endpointId, String rotationId, Collection regions) { + return new AssignedRotation( + new ClusterSpec.Id(clusterId), + new EndpointId(endpointId), + new RotationId(rotationId), + regions.stream().map(RegionName::from).collect(Collectors.toSet()) + ); + } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index 7390e0b2068..7b39b0d53a4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -121,7 +121,7 @@ public class ApplicationSerializerTest { OptionalInt.of(7), new MetricsService.ApplicationMetrics(0.5, 0.9), Optional.of("-----BEGIN PUBLIC KEY-----\n∠( ᐛ 」∠)＿\n-----END PUBLIC KEY-----"), - List.of(new AssignedRotation(new ClusterSpec.Id("foo"), EndpointId.default_(), new RotationId("my-rotation"), Set.of())), + List.of(AssignedRotation.fromStrings("foo", "default", "my-rotation", Set.of())), rotationStatus, Optional.of(new ApplicationCertificate("vespa.certificate"))); -- cgit v1.2.3 From 7e2bec9f5d8fb2ca503ded993a10d65e19f1d0f1 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 2 Jul 2019 11:25:13 +0200 Subject: Fix issue with rotations changing after assignment - Make sure we only include production zones in the assignment - Always run deployment through rotation assignment logic, so regions are cleared from persisted assignments even if there are no remaining rotations in the configuration. - Add tests to expose this logic. - Add tiny convenience method for AssignedRotation to create them in tests. --- .../hosted/controller/ApplicationController.java | 2 +- .../controller/rotation/RotationRepository.java | 78 ++++++++++++++++----- .../vespa/hosted/controller/ControllerTest.java | 81 ++++++++++++++++++++++ 3 files changed, 143 insertions(+), 18 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 0756d3006ea..ae8a81caa17 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -462,7 +462,7 @@ public class ApplicationController { /** Makes sure the application has a global rotation, if eligible. */ private LockedApplication withRotation(LockedApplication application, ZoneId zone) { - if (zone.environment() == Environment.prod && applicationNeedsRotations(application.get().deploymentSpec())) { + if (zone.environment() == Environment.prod) { try (RotationLock rotationLock = rotationRepository.lock()) { final var rotations = rotationRepository.getOrAssignRotations(application.get(), rotationLock); application = application.with(rotations); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index 7165d134d59..340a08e0ec6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -2,12 +2,14 @@ package com.yahoo.vespa.hosted.controller.rotation; import com.yahoo.collections.Pair; +import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.model.api.ContainerEndpoint; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.ApplicationController; import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; @@ -20,7 +22,9 @@ import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.function.Function; +import java.util.function.Predicate; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -96,6 +100,7 @@ public class RotationRepository { // Support the older case of setting global-service-id if (application.deploymentSpec().globalServiceId().isPresent()) { final var regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) .flatMap(zone -> zone.region().stream()) .collect(Collectors.toSet()); @@ -112,31 +117,70 @@ public class RotationRepository { } final var availableRotations = new ArrayList<>(availableRotations(lock).values()); - final var assignments = application.assignedRotations().stream() + final Map existingAssignments = existingEndpointAssignments(application); + final Map updatedAssignments = assignRotationsToEndpoints(application, existingAssignments, availableRotations); + + existingAssignments.putAll(updatedAssignments); + + return List.copyOf(existingAssignments.values()); + } + + private Map assignRotationsToEndpoints(Application application, Map existingAssignments, List availableRotations) { + return application.deploymentSpec().endpoints().stream() + .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .map(endpoint -> { + return new AssignedRotation( + new ClusterSpec.Id(endpoint.containerId()), + EndpointId.of(endpoint.endpointId()), + availableRotations.remove(0).id(), + endpoint.regions() + ); + }) .collect( Collectors.toMap( AssignedRotation::endpointId, Function.identity(), - (a, b) -> { throw new IllegalStateException("Duplicate entries: " + a + ", " + b); }, + (a, b) -> { throw new IllegalStateException("Duplicate entries:" + a + ", " + b); }, LinkedHashMap::new ) ); + } - application.deploymentSpec().endpoints().stream() - .filter(endpoint -> ! assignments.containsKey(new EndpointId(endpoint.endpointId()))) - .map(endpoint -> { - return new AssignedRotation( - new ClusterSpec.Id(endpoint.containerId()), - EndpointId.of(endpoint.endpointId()), - availableRotations.remove(0).id(), - endpoint.regions() - ); - }) - .forEach(assignment -> { - assignments.put(assignment.endpointId(), assignment); - }); + private Map existingEndpointAssignments(Application application) { + // + // Get the regions that has been configured for an endpoint. Empty set if the endpoint + // is no longer mentioned in the configuration file. + // + final Function> configuredRegionsForEndpoint = endpointId -> { + return application.deploymentSpec().endpoints().stream() + .filter(endpoint -> endpointId.id().equals(endpoint.endpointId())) + .map(Endpoint::regions) + .findFirst() + .orElse(Set.of()); + }; + + // + // Build a new AssignedRotation instance where we update set of regions from the configuration instead + // of using the one already mentioned in the assignment. This allows us to overwrite the set of regions + // when + final Function assignedRotationWithConfiguredRegions = assignedRotation -> { + return new AssignedRotation( + assignedRotation.clusterId(), + assignedRotation.endpointId(), + assignedRotation.rotationId(), + configuredRegionsForEndpoint.apply(assignedRotation.endpointId()) + ); + }; - return List.copyOf(assignments.values()); + return application.assignedRotations().stream() + .collect( + Collectors.toMap( + AssignedRotation::endpointId, + assignedRotationWithConfiguredRegions, + (a, b) -> { throw new IllegalStateException("Duplicate entries: " + a + ", " + b); }, + LinkedHashMap::new + ) + ); } /** diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 3a905f8c6d5..6e7ca423444 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -7,6 +7,7 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; +import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -23,6 +24,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevisi import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobError; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; @@ -344,6 +346,85 @@ public class ControllerTest { } @Test + public void testDnsAliasRegistrationWithChangingZones() { + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1", "us-central-1") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-central-1"))) + ); + + + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); + + assertEquals( + Set.of("rotation-id-01", "app1--tenant1.global.vespa.oath.cloud"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + + assertEquals( + Set.of(), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-central-1"))) + ); + + assertEquals(Set.of(RegionName.from("us-west-1")), tester.application(application.id()).assignedRotations().get(0).regions()); + } + + @Test + public void testUnassignRotations() { + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .endpoint("default", "qrs", "us-west-1", "us-central-1") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + + ApplicationPackage applicationPackage2 = new ApplicationPackageBuilder() + .environment(Environment.prod) + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); + + + assertEquals( + List.of(AssignedRotation.fromStrings("qrs", "default", "rotation-id-01", Set.of())), + tester.application(application.id()).assignedRotations() + ); + + assertEquals( + Set.of(), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1"))) + ); + } + + @Test public void testUpdatesExistingDnsAlias() { // Application 1 is deployed and deleted { -- cgit v1.2.3 From 145a2aae8786f4ee2c58b1452dcf5c15a4aa96a5 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 2 Jul 2019 12:52:12 +0200 Subject: Split finding needed rotations and assigning them to two steps - We find the needed rotations and check if we have enough before assigning, throwing a sensible exception if there are not enough. - Moved where we find available rotations so we are certain we have the lock during rotation assignment (this was guaranteed before, but now it is more explicit in the signature). - Added some javadoc to the new assignment method. --- .../controller/rotation/RotationRepository.java | 27 ++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java index 340a08e0ec6..f2bc50ec445 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java @@ -92,6 +92,17 @@ public class RotationRepository { return findAvailableRotation(application, lock); } + /** + * Returns rotation assignments for all endpoints in application. + * + * If rotations are already assigned, these will be returned. + * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. + * This method supports both global-service-id as well as the new endpoints tag. + * + * @param application The application requesting rotations. + * @param lock Lock which by acquired by the caller + * @return List of rotation assignments - either new or existing. + */ public List getOrAssignRotations(Application application, RotationLock lock) { if (application.deploymentSpec().globalServiceId().isPresent() && ! application.deploymentSpec().endpoints().isEmpty()) { throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); @@ -116,18 +127,26 @@ public class RotationRepository { ); } - final var availableRotations = new ArrayList<>(availableRotations(lock).values()); final Map existingAssignments = existingEndpointAssignments(application); - final Map updatedAssignments = assignRotationsToEndpoints(application, existingAssignments, availableRotations); + final Map updatedAssignments = assignRotationsToEndpoints(application, existingAssignments, lock); existingAssignments.putAll(updatedAssignments); return List.copyOf(existingAssignments.values()); } - private Map assignRotationsToEndpoints(Application application, Map existingAssignments, List availableRotations) { - return application.deploymentSpec().endpoints().stream() + private Map assignRotationsToEndpoints(Application application, Map existingAssignments, RotationLock lock) { + final var availableRotations = new ArrayList<>(availableRotations(lock).values()); + + final var neededRotations = application.deploymentSpec().endpoints().stream() .filter(Predicate.not(endpoint -> existingAssignments.containsKey(EndpointId.of(endpoint.endpointId())))) + .collect(Collectors.toSet()); + + if (neededRotations.size() > availableRotations.size()) { + throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation: need " + neededRotations.size() + ", have " + availableRotations.size()); + } + + return neededRotations.stream() .map(endpoint -> { return new AssignedRotation( new ClusterSpec.Id(endpoint.containerId()), -- cgit v1.2.3 From 425b6dea836cf5f76987f232049eab4d0e15d06a Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 2 Jul 2019 13:00:03 +0200 Subject: Rename .defaultGlobal => .create --- .../main/java/com/yahoo/vespa/hosted/controller/Application.java | 6 ++---- .../com/yahoo/vespa/hosted/controller/application/EndpointList.java | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java index 1b6b5a6fb5b..d8b56502fc3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Application.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; @@ -21,7 +20,6 @@ import com.yahoo.vespa.hosted.controller.application.AssignedRotation; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentJobs; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.RotationStatus; @@ -224,14 +222,14 @@ public class Application { /** Returns the default global endpoints for this in given system - for a given endpoint ID */ public EndpointList endpointsIn(SystemName system, EndpointId endpointId) { if (rotations.isEmpty()) return EndpointList.EMPTY; - return EndpointList.defaultGlobal(id, endpointId, system); + return EndpointList.create(id, endpointId, system); } /** Returns the default global endpoints for this in given system */ public EndpointList endpointsIn(SystemName system) { if (rotations.isEmpty()) return EndpointList.EMPTY; final var endpointStream = rotations.stream() - .flatMap(rotation -> EndpointList.defaultGlobal(id, rotation.endpointId(), system).asList().stream()); + .flatMap(rotation -> EndpointList.create(id, rotation.endpointId(), system).asList().stream()); return EndpointList.of(endpointStream); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 216b6c1eab9..feedc1c8f9d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -67,7 +67,7 @@ public class EndpointList { } /** Returns the default global endpoints in given system. Default endpoints are served by a pre-provisioned routing layer */ - public static EndpointList defaultGlobal(ApplicationId application, EndpointId endpointId, SystemName system) { + public static EndpointList create(ApplicationId application, EndpointId endpointId, SystemName system) { switch (system) { case cd: case main: -- cgit v1.2.3 From 3f176936be8b6a33da51238bc35c42cdae24881a Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Tue, 2 Jul 2019 15:01:28 +0200 Subject: Introduce feature flag on application ID for new endpoints behavior - Added a feature flag on application level for new endpoints behavior - Reintroduced a test of legacy behavior to validate old logic --- .../hosted/controller/ApplicationController.java | 108 ++++++++++++++++----- .../vespa/hosted/controller/ControllerTest.java | 57 ++++++++++- .../controller/integration/ConfigServerMock.java | 9 +- .../maintenance/MetricsReporterTest.java | 2 +- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ++ 5 files changed, 153 insertions(+), 29 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index ae8a81caa17..d015e65a5e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -2,6 +2,7 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; +import com.yahoo.collections.ArraySet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -9,6 +10,7 @@ import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.athenz.api.AthenzDomain; @@ -129,6 +131,7 @@ public class ApplicationController { private final RoutingPolicies routingPolicies; private final Clock clock; private final BooleanFlag redirectLegacyDnsFlag; + private final BooleanFlag useMultipleEndpoints; private final DeploymentTrigger deploymentTrigger; private final BooleanFlag provisionApplicationCertificate; private final ApplicationCertificateProvider applicationCertificateProvider; @@ -146,6 +149,7 @@ public class ApplicationController { this.routingPolicies = new RoutingPolicies(controller); this.clock = clock; this.redirectLegacyDnsFlag = Flags.REDIRECT_LEGACY_DNS_NAMES.bindTo(controller.flagSource()); + this.useMultipleEndpoints = Flags.MULTIPLE_GLOBAL_ENDPOINTS.bindTo(controller.flagSource()); this.artifactRepository = artifactRepository; this.applicationStore = applicationStore; @@ -296,6 +300,7 @@ public class ApplicationController { Version platformVersion; ApplicationVersion applicationVersion; ApplicationPackage applicationPackage; + Set legacyRotations = new LinkedHashSet<>(); Set endpoints = new LinkedHashSet<>(); ApplicationCertificate applicationCertificate; @@ -335,22 +340,34 @@ public class ApplicationController { // TODO: Remove this when all packages are validated upon submission, as in ApplicationApiHandler.submit(...). verifyApplicationIdentityConfiguration(applicationId.tenant(), applicationPackage, deployingIdentity); + // Assign global rotation - application = withRotation(application, zone); - Application app = application.get(); - // Include global DNS names - app.assignedRotations().stream() - .filter(assignedRotation -> assignedRotation.regions().contains(zone.region())) - .map(assignedRotation -> { - return new ContainerEndpoint( - assignedRotation.clusterId().value(), - Stream.concat( - app.endpointsIn(controller.system(), assignedRotation.endpointId()).legacy(false).asList().stream().map(Endpoint::dnsName), - app.rotations().stream().map(RotationId::asString) - ).collect(Collectors.toList()) - ); - }) - .forEach(endpoints::add); + if (useMultipleEndpoints.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()).value()) { + application = withRotation(application, zone); + + // Include global DNS names + Application app = application.get(); + app.assignedRotations().stream() + .filter(assignedRotation -> assignedRotation.regions().contains(zone.region())) + .map(assignedRotation -> { + return new ContainerEndpoint( + assignedRotation.clusterId().value(), + Stream.concat( + app.endpointsIn(controller.system(), assignedRotation.endpointId()).legacy(false).asList().stream().map(Endpoint::dnsName), + app.rotations().stream().map(RotationId::asString) + ).collect(Collectors.toList()) + ); + }) + .forEach(endpoints::add); + } else { + application = withRotationLegacy(application, zone); + + // Add both the names we have in DNS for each endpoint as well as name of the rotation so healthchecks works + Application app = application.get(); + app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(legacyRotations::add); + app.rotations().stream().map(RotationId::asString).forEach(legacyRotations::add); + } + // Get application certificate (provisions a new certificate if missing) application = withApplicationCertificate(application); @@ -366,7 +383,7 @@ public class ApplicationController { // Carry out deployment without holding the application lock. options = withVersion(platformVersion, options); - ActivateResult result = deploy(applicationId, applicationPackage, zone, options, endpoints, applicationCertificate); + ActivateResult result = deploy(applicationId, applicationPackage, zone, options, legacyRotations, endpoints, applicationCertificate); lockOrThrow(applicationId, application -> store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(), @@ -433,7 +450,7 @@ public class ApplicationController { artifactRepository.getSystemApplicationPackage(application.id(), zone, version) ); DeployOptions options = withVersion(version, DeployOptions.none()); - return deploy(application.id(), applicationPackage, zone, options, Set.of(), /* No application cert */ null); + return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert */ null); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -441,16 +458,16 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) { - return deploy(tester.id(), applicationPackage, zone, options, Set.of(), /* No application cert for tester*/ null); + return deploy(tester.id(), applicationPackage, zone, options, Set.of(), Set.of(), /* No application cert for tester*/ null); } private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions deployOptions, - Set endpoints, ApplicationCertificate applicationCertificate) { + Set legacyRotations, Set endpoints, ApplicationCertificate applicationCertificate) { DeploymentId deploymentId = new DeploymentId(application, zone); try { ConfigServer.PreparedApplication preparedApplication = - configServer.deploy(deploymentId, deployOptions, Set.of(), endpoints, applicationCertificate, applicationPackage.zippedContent()); + configServer.deploy(deploymentId, deployOptions, legacyRotations, endpoints, applicationCertificate, applicationPackage.zippedContent()); return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { @@ -460,6 +477,53 @@ public class ApplicationController { } } + /** Makes sure the application has a global rotation, if eligible. */ + private LockedApplication withRotationLegacy(LockedApplication application, ZoneId zone) { + if (zone.environment() == Environment.prod && application.get().deploymentSpec().globalServiceId().isPresent()) { + try (RotationLock rotationLock = rotationRepository.lock()) { + Rotation rotation = rotationRepository.getOrAssignRotation(application.get(), rotationLock); + application = application.with(createDefaultGlobalIdRotation(application.get(), rotation)); + store(application); // store assigned rotation even if deployment fails + + boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) + .value(); + + EndpointList globalEndpoints = application.get() + .endpointsIn(controller.system()) + .scope(Endpoint.Scope.global); + + globalEndpoints.main().ifPresent(mainEndpoint -> { + registerCname(mainEndpoint.dnsName(), rotation.name()); + if (redirectLegacyDns) { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); + } else { + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); + } + }); + } + } + return application; + } + + private List createDefaultGlobalIdRotation(Application application, Rotation rotation) { + // This is guaranteed by .withRotationLegacy, but add this to make inspections accept the use of .get() below + assert application.deploymentSpec().globalServiceId().isPresent(); + + final Set regions = application.deploymentSpec().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + + final var assignment = new AssignedRotation( + ClusterSpec.Id.from(application.deploymentSpec().globalServiceId().get()), + EndpointId.default_(), + rotation.id(), + regions + ); + + return List.of(assignment); + } + /** Makes sure the application has a global rotation, if eligible. */ private LockedApplication withRotation(LockedApplication application, ZoneId zone) { if (zone.environment() == Environment.prod) { @@ -473,10 +537,6 @@ public class ApplicationController { return application; } - private boolean applicationNeedsRotations(DeploymentSpec spec) { - return spec.globalServiceId().isPresent() || !spec.endpoints().isEmpty(); - } - private void registerAssignedRotationCnames(Application application) { application.assignedRotations().forEach(assignedRotation -> { final var endpoints = application diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 6e7ca423444..c26f1879f6a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -277,6 +277,8 @@ public class ControllerTest { @Test public void testDnsAliasRegistration() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -305,8 +307,51 @@ public class ControllerTest { assertEquals("rotation-fqdn-01.", record.get().data().asString()); } + @Test + public void testDnsAliasRegistrationLegacy() { + Application application = tester.createApplication("app1", "tenant1", 1, 1L); + + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .environment(Environment.prod) + .globalServiceId("foo") + .region("us-west-1") + .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .build(); + + tester.deployCompletely(application, applicationPackage); + Collection deployments = tester.application(application.id()).deployments().values(); + assertFalse(deployments.isEmpty()); + for (Deployment deployment : deployments) { + assertEquals("Rotation names are passed to config server in " + deployment.zone(), + Set.of("rotation-id-01", + "app1--tenant1.global.vespa.oath.cloud", + "app1.tenant1.global.vespa.yahooapis.com", + "app1--tenant1.global.vespa.yahooapis.com"), + tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); + } + tester.flushDnsRequests(); + assertEquals(3, tester.controllerTester().nameService().records().size()); + + Optional record = tester.controllerTester().findCname("app1--tenant1.global.vespa.yahooapis.com"); + assertTrue(record.isPresent()); + assertEquals("app1--tenant1.global.vespa.yahooapis.com", record.get().name().asString()); + assertEquals("rotation-fqdn-01.", record.get().data().asString()); + + record = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record.isPresent()); + assertEquals("app1--tenant1.global.vespa.oath.cloud", record.get().name().asString()); + assertEquals("rotation-fqdn-01.", record.get().data().asString()); + + record = tester.controllerTester().findCname("app1.tenant1.global.vespa.yahooapis.com"); + assertTrue(record.isPresent()); + assertEquals("app1.tenant1.global.vespa.yahooapis.com", record.get().name().asString()); + assertEquals("rotation-fqdn-01.", record.get().data().asString()); + } + @Test public void testDnsAliasRegistrationWithEndpoints() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -314,7 +359,7 @@ public class ControllerTest { .endpoint("foobar", "qrs", "us-west-1", "us-central-1") .endpoint("default", "qrs", "us-west-1", "us-central-1") .region("us-west-1") - .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .region("us-central-1") .build(); tester.deployCompletely(application, applicationPackage); @@ -347,13 +392,15 @@ public class ControllerTest { @Test public void testDnsAliasRegistrationWithChangingZones() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() .environment(Environment.prod) .endpoint("default", "qrs", "us-west-1", "us-central-1") .region("us-west-1") - .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .region("us-central-1") .build(); tester.deployCompletely(application, applicationPackage); @@ -373,7 +420,7 @@ public class ControllerTest { .environment(Environment.prod) .endpoint("default", "qrs", "us-west-1") .region("us-west-1") - .region("us-central-1") // Two deployments should result in each DNS alias being registered once + .region("us-central-1") .build(); tester.deployCompletely(application, applicationPackage2, BuildJob.defaultBuildNumber + 1); @@ -393,6 +440,8 @@ public class ControllerTest { @Test public void testUnassignRotations() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + Application application = tester.createApplication("app1", "tenant1", 1, 1L); ApplicationPackage applicationPackage = new ApplicationPackageBuilder() @@ -426,6 +475,8 @@ public class ControllerTest { @Test public void testUpdatesExistingDnsAlias() { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.MULTIPLE_GLOBAL_ENDPOINTS.id(), true); + // Application 1 is deployed and deleted { Application app1 = tester.createApplication("app1", "tenant1", 1, 1L); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 4841b6c0c12..a89c5988396 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -45,6 +45,7 @@ import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; +import java.util.stream.Stream; /** * @author mortent @@ -238,7 +239,13 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty()) provision(deployment.zoneId(), deployment.applicationId()); - this.rotationNames.put(deployment, containerEndpoints.stream().flatMap(e -> e.names().stream()).collect(Collectors.toSet())); + this.rotationNames.put( + deployment, + Stream.concat( + containerEndpoints.stream().flatMap(e -> e.names().stream()), + rotationNames.stream() + ).collect(Collectors.toSet()) + ); return () -> { Application application = applications.get(deployment.applicationId()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index b994c55ce55..58f35c0ac05 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -253,7 +253,7 @@ public class MetricsReporterTest { tester.deployCompletely(application, applicationPackage); reporter.maintain(); - assertEquals("Deployment queues name services requests", 2, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); + assertEquals("Deployment queues name services requests", 6, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); tester.flushDnsRequests(); reporter.maintain(); diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index b969e328419..5e9c0e2543c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -151,6 +151,12 @@ public class Flags { "Takes effect on deployment through controller", APPLICATION_ID); + public static final UnboundBooleanFlag MULTIPLE_GLOBAL_ENDPOINTS = defineFeatureFlag( + "multiple-global-endpoints", false, + "Allow applications to use new endpoints syntax in deployment.xml", + "Takes effect on deployment through controller", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { -- cgit v1.2.3 From 8558e33b1d202e8d5a44ac1920f5735b1742aaf0 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 12 Jun 2019 15:15:57 +0200 Subject: Remove promote API --- .../hosted/controller/api/role/PathGroup.java | 4 +- .../yahoo/vespa/hosted/controller/Controller.java | 13 ++----- .../restapi/application/ApplicationApiHandler.java | 40 +------------------- .../application/ApplicationChefEnvironment.java | 43 ---------------------- .../vespa/hosted/controller/ControllerTester.java | 13 +++---- .../restapi/application/ApplicationApiTest.java | 2 - 6 files changed, 10 insertions(+), 105 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 6745b75a9ea..1ec75e0c998 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -141,9 +141,7 @@ enum PathGroup { Matcher.application, Optional.of("/api"), "/application/v4/tenant/{tenant}/application/{application}/jobreport", - "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport", - "/application/v4/tenant/{tenant}/application/{application}/promote", - "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote"), + "/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/jobreport"), /** Paths which contain (not very strictly) classified information about customers. */ classifiedTenantInfo(Optional.of("/api"), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index ed81d08c533..08c95d1ecab 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -15,7 +15,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.BuildService; import com.yahoo.vespa.hosted.controller.api.integration.MetricsService; import com.yahoo.vespa.hosted.controller.api.integration.RunDataStore; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificateProvider; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -79,7 +78,6 @@ public class Controller extends AbstractComponent { private final ZoneRegistry zoneRegistry; private final ConfigServer configServer; private final MetricsService metricsService; - private final Chef chef; private final Mailer mailer; private final AuditLogger auditLogger; private final FlagSource flagSource; @@ -95,13 +93,13 @@ public class Controller extends AbstractComponent { @Inject public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, + RoutingGenerator routingGenerator, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Mailer mailer, FlagSource flagSource, MavenRepository mavenRepository, ApplicationCertificateProvider applicationCertificateProvider) { this(curator, rotationsConfig, gitHub, zoneRegistry, - configServer, metricsService, routingGenerator, chef, + configServer, metricsService, routingGenerator, Clock.systemUTC(), accessControl, artifactRepository, applicationStore, testerCloud, buildService, runDataStore, com.yahoo.net.HostName::getLocalhost, mailer, flagSource, mavenRepository, applicationCertificateProvider); @@ -110,7 +108,7 @@ public class Controller extends AbstractComponent { public Controller(CuratorDb curator, RotationsConfig rotationsConfig, GitHub gitHub, ZoneRegistry zoneRegistry, ConfigServer configServer, MetricsService metricsService, - RoutingGenerator routingGenerator, Chef chef, Clock clock, + RoutingGenerator routingGenerator, Clock clock, AccessControl accessControl, ArtifactRepository artifactRepository, ApplicationStore applicationStore, TesterCloud testerCloud, BuildService buildService, RunDataStore runDataStore, Supplier hostnameSupplier, @@ -122,7 +120,6 @@ public class Controller extends AbstractComponent { this.zoneRegistry = Objects.requireNonNull(zoneRegistry, "ZoneRegistry cannot be null"); this.configServer = Objects.requireNonNull(configServer, "ConfigServer cannot be null"); this.metricsService = Objects.requireNonNull(metricsService, "MetricsService cannot be null"); - this.chef = Objects.requireNonNull(chef, "Chef cannot be null"); this.clock = Objects.requireNonNull(clock, "Clock cannot be null"); this.mailer = Objects.requireNonNull(mailer, "Mailer cannot be null"); this.flagSource = Objects.requireNonNull(flagSource, "FlagSource cannot be null"); @@ -294,10 +291,6 @@ public class Controller extends AbstractComponent { return zoneRegistry.system(); } - public Chef chefClient() { - return chef; - } - public CuratorDb curator() { return curator; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9f091061596..dce7d2e68ac 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -7,15 +7,13 @@ import com.google.common.collect.ImmutableSet; import com.google.inject.Inject; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.io.IOUtils; -import com.yahoo.log.LogLevel; import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; @@ -50,7 +48,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterCost; @@ -220,7 +217,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handlePOST(Path path, HttpRequest request) { if (path.matches("/application/v4/tenant/{tenant}")) return createTenant(path.get("tenant"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return createApplication(path.get("tenant"), path.get("application"), "default", request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/promote")) return promoteApplication(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/platform")) return deployPlatform(path.get("tenant"), path.get("application"), "default", false, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deployPlatform(path.get("tenant"), path.get("application"), "default", true, request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/application")) return deployApplication(path.get("tenant"), path.get("application"), "default", request); @@ -238,11 +234,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/deploy")) return deploy(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); // legacy synonym of the above if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/restart")) return restart(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request); - if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/promote")) return promoteApplicationDeployment(path.get("tenant"), path.get("application"), path.get("environment"), path.get("region"), path.get("instance"), request); return ErrorResponse.notFoundError("Nothing at " + path); } @@ -1120,38 +1114,6 @@ public class ApplicationApiHandler extends LoggingRequestHandler { "region", region)); } - /** - * Promote application Chef environments. To be used by component jobs only - */ - private HttpResponse promoteApplication(String tenantName, String applicationName, HttpRequest request) { - try{ - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.systemChefEnvironment(); - String targetEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s)", tenantName, applicationName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } - } - - /** - * Promote application Chef environments for jobs that deploy applications - */ - private HttpResponse promoteApplicationDeployment(String tenantName, String applicationName, String environmentName, String regionName, String instanceName, HttpRequest request) { - try { - ApplicationChefEnvironment chefEnvironment = new ApplicationChefEnvironment(controller.system()); - String sourceEnvironment = chefEnvironment.applicationSourceEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName)); - String targetEnvironment = chefEnvironment.applicationTargetEnvironment(TenantName.from(tenantName), ApplicationName.from(applicationName), Environment.from(environmentName), RegionName.from(regionName)); - controller.chefClient().copyChefEnvironment(sourceEnvironment, targetEnvironment); - return new MessageResponse(String.format("Successfully copied environment %s to %s", sourceEnvironment, targetEnvironment)); - } catch (Exception e) { - log.log(LogLevel.ERROR, String.format("Error during Chef copy environment. (%s.%s %s.%s)", tenantName, applicationName, environmentName, regionName), e); - return ErrorResponse.internalServerError("Unable to promote Chef environments for application"); - } - } - private HttpResponse notifyJobCompletion(String tenant, String application, HttpRequest request) { try { DeploymentJobs.JobReport report = toJobReport(tenant, application, toSlime(request.getData()).get()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java deleted file mode 100644 index 7c32e48e218..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationChefEnvironment.java +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.TenantName; - -/** - * Represents Chef environments for applications/deployments. Used for promotion of Chef environments - * - * @author mortent - */ -public class ApplicationChefEnvironment { - - private final String systemChefEnvironment; - private final String systemSuffix; - - public ApplicationChefEnvironment(SystemName system) { - if (system == SystemName.main) { - systemChefEnvironment = "hosted-verified-prod"; - systemSuffix = ""; - } else { - systemChefEnvironment = "hosted-infra-cd"; - systemSuffix = "-cd"; - } - } - - public String systemChefEnvironment() { - return systemChefEnvironment; - } - - public String applicationSourceEnvironment(TenantName tenantName, ApplicationName applicationName) { - // placeholder and component already used in legacy chef promotion - return String.format("hosted-instance%s_%s_%s_placeholder_component_default", systemSuffix, tenantName, applicationName); - } - - public String applicationTargetEnvironment(TenantName tenantName, ApplicationName applicationName, Environment environment, RegionName regionName) { - return String.format("hosted-instance%s_%s_%s_%s_%s_default", systemSuffix, tenantName, applicationName, regionName, environment); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index dbf983a5bab..d5935c752d9 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -19,7 +19,6 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.BuildService; -import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -29,7 +28,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.api.integration.github.GitHubMock; import com.yahoo.vespa.hosted.controller.api.integration.organization.Contact; import com.yahoo.vespa.hosted.controller.api.integration.organization.MockContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.organization.MockIssueHandler; import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockBuildService; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; @@ -100,7 +98,7 @@ public final class ControllerTester { this(new AthenzDbMock(), clock, new ConfigServerMock(new ZoneRegistryMock()), new ZoneRegistryMock(), new GitHubMock(), curatorDb, rotationsConfig, new MemoryNameService(), new ArtifactRepositoryMock(), new ApplicationStoreMock(), new MockBuildService(), - metricsService, new RoutingGeneratorMock(), new MockContactRetriever(), new MockIssueHandler(clock)); + metricsService, new RoutingGeneratorMock(), new MockContactRetriever()); } public ControllerTester(ManualClock clock) { @@ -125,7 +123,7 @@ public final class ControllerTester { MemoryNameService nameService, ArtifactRepositoryMock artifactRepository, ApplicationStoreMock appStoreMock, MockBuildService buildService, MetricsServiceMock metricsService, RoutingGeneratorMock routingGenerator, - MockContactRetriever contactRetriever, MockIssueHandler issueHandler) { + MockContactRetriever contactRetriever) { this.athenzDb = athenzDb; this.clock = clock; this.configServer = configServer; @@ -141,7 +139,7 @@ public final class ControllerTester { this.routingGenerator = routingGenerator; this.contactRetriever = contactRetriever; this.controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, - athenzDb, nameService, artifactRepository, appStoreMock, buildService, + athenzDb, artifactRepository, appStoreMock, buildService, metricsService, routingGenerator); // Make root logger use time from manual clock @@ -199,7 +197,7 @@ public final class ControllerTester { /** Create a new controller instance. Useful to verify that controller state is rebuilt from persistence */ public final void createNewController() { controller = createController(curator, rotationsConfig, configServer, clock, gitHub, zoneRegistry, athenzDb, - nameService, artifactRepository, applicationStore, buildService, metricsService, + artifactRepository, applicationStore, buildService, metricsService, routingGenerator); } @@ -332,7 +330,7 @@ public final class ControllerTester { private static Controller createController(CuratorDb curator, RotationsConfig rotationsConfig, ConfigServerMock configServer, ManualClock clock, GitHubMock gitHub, ZoneRegistryMock zoneRegistryMock, - AthenzDbMock athensDb, MemoryNameService nameService, + AthenzDbMock athensDb, ArtifactRepository artifactRepository, ApplicationStore applicationStore, BuildService buildService, MetricsServiceMock metricsService, RoutingGenerator routingGenerator) { @@ -343,7 +341,6 @@ public final class ControllerTester { configServer, metricsService, routingGenerator, - new ChefMock(), clock, new AthenzFacade(new AthenzClientFactoryMock(athensDb)), artifactRepository, diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 16fd10277d2..a0acbc625f7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -16,7 +16,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.athenz.api.AthenzDomain; import com.yahoo.vespa.athenz.api.AthenzIdentity; import com.yahoo.vespa.athenz.api.AthenzUser; @@ -61,7 +60,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ContainerControllerTester; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; -import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import org.junit.Before; -- cgit v1.2.3 From 095b2c3ca4f0e88263bd9971da68c53950b66093 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 3 Jul 2019 10:58:45 +0200 Subject: Stop reporting Chef convergence metric --- .../maintenance/ControllerMaintenance.java | 13 ++- .../controller/maintenance/MetricsReporter.java | 70 +------------- .../maintenance/MetricsReporterTest.java | 104 ++------------------- .../restapi/ControllerContainerTest.java | 1 - .../src/test/resources/chef_output.json | 34 ------- 5 files changed, 18 insertions(+), 204 deletions(-) delete mode 100644 controller-server/src/test/resources/chef_output.json (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index e840deb062c..f34c24c497a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,15 +5,14 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; -import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; +import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; import com.yahoo.vespa.hosted.controller.api.integration.organization.OwnershipIssues; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; +import com.yahoo.vespa.hosted.controller.authority.config.ApiAuthorityConfig; import com.yahoo.vespa.hosted.controller.maintenance.config.MaintainerConfig; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; @@ -59,7 +58,7 @@ public class ControllerMaintenance extends AbstractComponent { @SuppressWarnings("unused") // instantiated by Dependency Injection public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, - JobControl jobControl, Metric metric, Chef chefClient, + JobControl jobControl, Metric metric, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, ContactRetriever contactRetriever, @@ -71,7 +70,7 @@ public class ControllerMaintenance extends AbstractComponent { this.jobControl = jobControl; deploymentExpirer = new DeploymentExpirer(controller, maintenanceInterval, jobControl); deploymentIssueReporter = new DeploymentIssueReporter(controller, deploymentIssues, maintenanceInterval, jobControl); - metricsReporter = new MetricsReporter(controller, metric, chefClient, jobControl, controller.system()); + metricsReporter = new MetricsReporter(controller, metric, jobControl); outstandingChangeDeployer = new OutstandingChangeDeployer(controller, Duration.ofMinutes(1), jobControl); versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 5e3f21c6b98..c7b76696d84 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -3,14 +3,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.SystemName; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.chef.AttributeMapping; -import com.yahoo.vespa.hosted.controller.api.integration.chef.Chef; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNode; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -24,10 +19,8 @@ import java.time.Duration; import java.time.Instant; import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.stream.Collectors; /** @@ -36,7 +29,6 @@ import java.util.stream.Collectors; */ public class MetricsReporter extends Maintainer { - public static final String CONVERGENCE_METRIC = "seconds.since.last.chef.convergence"; public static final String DEPLOYMENT_FAIL_METRIC = "deployment.failurePercentage"; public static final String DEPLOYMENT_AVERAGE_DURATION = "deployment.averageDuration"; public static final String DEPLOYMENT_FAILING_UPGRADES = "deployment.failingUpgrades"; @@ -46,27 +38,16 @@ public class MetricsReporter extends Maintainer { public static final String NAME_SERVICE_REQUESTS_QUEUED = "dns.queuedRequests"; private final Metric metric; - private final Chef chefClient; private final Clock clock; - private final SystemName system; - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, JobControl jobControl, - SystemName system) { - this(controller, metric, chefClient, Clock.systemUTC(), jobControl, system); - } - - public MetricsReporter(Controller controller, Metric metric, Chef chefClient, Clock clock, - JobControl jobControl, SystemName system) { + public MetricsReporter(Controller controller, Metric metric, JobControl jobControl) { super(controller, Duration.ofMinutes(1), jobControl); // use fixed rate for metrics this.metric = metric; - this.chefClient = chefClient; - this.clock = clock; - this.system = system; + this.clock = controller.clock(); } @Override public void maintain() { - reportChefMetrics(); reportDeploymentMetrics(); reportRemainingRotations(); reportQueuedNameServiceRequests(); @@ -79,49 +60,6 @@ public class MetricsReporter extends Maintainer { } } - private void reportChefMetrics() { - String query = "chef_environment:hosted*"; - if (system == SystemName.cd) { - query += " AND hosted_system:" + system; - } - PartialNodeResult nodeResult = chefClient.partialSearchNodes(query, - List.of( - AttributeMapping.simpleMapping("fqdn"), - AttributeMapping.simpleMapping("ohai_time"), - AttributeMapping.deepMapping("tenant", List.of("hosted", "owner", "tenant")), - AttributeMapping.deepMapping("application", List.of("hosted", "owner", "application")), - AttributeMapping.deepMapping("instance", List.of("hosted", "owner", "instance")), - AttributeMapping.deepMapping("environment", List.of("hosted", "environment")), - AttributeMapping.deepMapping("region", List.of("hosted", "region")), - AttributeMapping.deepMapping("system", List.of("hosted", "system")) - )); - - // The above search will return a correct list if the system is CD. However for main, it will - // return all nodes, since system==nil for main - keepNodesWithSystem(nodeResult, system); - - Instant instant = clock.instant(); - for (PartialNode node : nodeResult.rows) { - String hostname = node.getFqdn(); - long secondsSinceConverge = Duration.between(Instant.ofEpochSecond(node.getOhaiTime().longValue()), instant).getSeconds(); - Map dimensions = new HashMap<>(); - dimensions.put("host", hostname); - dimensions.put("system", node.getValue("system").orElse("main")); - Optional environment = node.getValue("environment"); - Optional region = node.getValue("region"); - - if (environment.isPresent() && region.isPresent()) { - dimensions.put("zone", String.format("%s.%s", environment.get(), region.get())); - } - - node.getValue("tenant").ifPresent(tenant -> dimensions.put("tenantName", tenant)); - Optional application = node.getValue("application"); - application.ifPresent(app -> dimensions.put("app", String.format("%s.%s", app, node.getValue("instance").orElse("default")))); - Metric.Context context = metric.createContext(dimensions); - metric.set(CONVERGENCE_METRIC, secondsSinceConverge, context); - } - } - private void reportDeploymentMetrics() { ApplicationList applications = ApplicationList.from(controller().applications().asList()) .hasProductionDeployment(); @@ -210,10 +148,6 @@ public class MetricsReporter extends Maintainer { .max(Integer::compareTo) .orElse(0); } - - private static void keepNodesWithSystem(PartialNodeResult nodeResult, SystemName system) { - nodeResult.rows.removeIf(node -> !system.value().equals(node.getValue("system").orElse("main"))); - } private static Map dimensions(ApplicationId application) { return ImmutableMap.of( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java index 58f35c0ac05..148be3f258e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporterTest.java @@ -1,38 +1,23 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; import com.yahoo.component.Version; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.SystemName; -import com.yahoo.test.ManualClock; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.chef.ChefMock; -import com.yahoo.vespa.hosted.controller.api.integration.chef.rest.PartialNodeResult; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; -import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester; import com.yahoo.vespa.hosted.controller.integration.MetricsMock; -import com.yahoo.vespa.hosted.controller.integration.MetricsMock.MapContext; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import org.junit.Before; import org.junit.Test; -import java.io.IOException; -import java.io.UncheckedIOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Clock; import java.time.Duration; -import java.time.Instant; -import java.util.Map; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.component; import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; @@ -40,39 +25,13 @@ import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobTy import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNull; /** * @author mortent */ public class MetricsReporterTest { - private static final Path testData = Paths.get("src/test/resources/"); - - private MetricsMock metrics; - - @Before - public void before() { - metrics = new MetricsMock(); - } - - @Test - public void test_chef_metrics() { - Clock clock = new ManualClock(Instant.ofEpochSecond(1475497913)); - ControllerTester tester = new ControllerTester(); - MetricsReporter metricsReporter = createReporter(clock, tester.controller(), metrics, SystemName.cd); - metricsReporter.maintain(); - assertEquals(2, metrics.getMetrics().size()); - - Map> hostMetrics = getMetricsByHost("fake-node.test"); - assertEquals(1, hostMetrics.size()); - Map.Entry> metricEntry = hostMetrics.entrySet().iterator().next(); - MapContext metricContext = metricEntry.getKey(); - assertDimension(metricContext, "tenantName", "ciintegrationtests"); - assertDimension(metricContext, "app", "restart.default"); - assertDimension(metricContext, "zone", "prod.cd-us-east-1"); - assertEquals(727, metricEntry.getValue().get(MetricsReporter.CONVERGENCE_METRIC).longValue()); - } + private final MetricsMock metrics = new MetricsMock(); @Test public void test_deployment_fail_ratio() { @@ -81,7 +40,7 @@ public class MetricsReporterTest { .environment(Environment.prod) .region("us-west-1") .build(); - MetricsReporter metricsReporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter metricsReporter = createReporter(tester.controller()); metricsReporter.maintain(); assertEquals(0.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); @@ -107,14 +66,6 @@ public class MetricsReporterTest { assertEquals(25.0, metrics.getMetric(MetricsReporter.DEPLOYMENT_FAIL_METRIC)); } - @Test - public void test_chef_metrics_omit_zone_when_unknown() { - ControllerTester tester = new ControllerTester(); - String hostname = "fake-node2.test"; - MapContext metricContext = getMetricContextByHost(tester.controller(), hostname); - assertNull(metricContext.getDimensions().get("zone")); - } - @Test public void test_deployment_average_duration() { DeploymentTester tester = new DeploymentTester(); @@ -123,7 +74,7 @@ public class MetricsReporterTest { .region("us-west-1") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application app = tester.createApplication("app1", "tenant1", 1, 11L); tester.deployCompletely(app, applicationPackage); @@ -165,7 +116,7 @@ public class MetricsReporterTest { .region("us-west-1") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application app = tester.createApplication("app1", "tenant1", 1, 11L); // Initial deployment without failures @@ -216,7 +167,7 @@ public class MetricsReporterTest { .region("us-west-1") .region("us-east-3") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 11L); tester.configServer().generateWarnings(new DeploymentId(application.id(), ZoneId.from("prod", "us-west-1")), 3); tester.configServer().generateWarnings(new DeploymentId(application.id(), ZoneId.from("prod", "us-east-3")), 4); @@ -231,7 +182,7 @@ public class MetricsReporterTest { ApplicationVersion version = tester.deployNewSubmission(); assertEquals(1000, version.buildTime().get().toEpochMilli()); - MetricsReporter reporter = createReporter(tester.tester().controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.tester().controller()); reporter.maintain(); assertEquals(tester.clock().instant().getEpochSecond() - 1, getMetric(MetricsReporter.DEPLOYMENT_BUILD_AGE_SECONDS, tester.app())); @@ -246,7 +197,7 @@ public class MetricsReporterTest { .region("us-west-1") .region("us-east-3") .build(); - MetricsReporter reporter = createReporter(tester.controller(), metrics, SystemName.main); + MetricsReporter reporter = createReporter(tester.controller()); Application application = tester.createApplication("app1", "tenant1", 1, 11L); reporter.maintain(); assertEquals("Queue is empty initially", 0, metrics.getMetric(MetricsReporter.NAME_SERVICE_REQUESTS_QUEUED).intValue()); @@ -279,43 +230,8 @@ public class MetricsReporterTest { .orElseThrow(() -> new RuntimeException("Expected metric to exist for " + application.id())); } - private MetricsReporter createReporter(Controller controller, MetricsMock metricsMock, SystemName system) { - return createReporter(controller.clock(), controller, metricsMock, system); - } - - private MetricsReporter createReporter(Clock clock, Controller controller, MetricsMock metricsMock, - SystemName system) { - ChefMock chef = new ChefMock(); - PartialNodeResult result; - try { - result = new ObjectMapper() - .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false) - .readValue(testData.resolve("chef_output.json").toFile(), PartialNodeResult.class); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - chef.addPartialResult(result.rows); - return new MetricsReporter(controller, metricsMock, chef, clock, new JobControl(new MockCuratorDb()), system); - } - - private Map> getMetricsByHost(String hostname) { - return metrics.getMetrics((dimensions) -> hostname.equals(dimensions.get("host"))); - } - - private MapContext getMetricContextByHost(Controller controller, String hostname) { - MetricsReporter metricsReporter = createReporter(controller, metrics, SystemName.main); - metricsReporter.maintain(); - - assertFalse(metrics.getMetrics().isEmpty()); - - Map> metrics = getMetricsByHost(hostname); - assertEquals(1, metrics.size()); - Map.Entry> metricEntry = metrics.entrySet().iterator().next(); - return metricEntry.getKey(); - } - - private static void assertDimension(MapContext metricContext, String dimensionName, String expectedValue) { - assertEquals(expectedValue, metricContext.getDimensions().get(dimensionName)); + private MetricsReporter createReporter(Controller controller) { + return new MetricsReporter(controller, metrics, new JobControl(new MockCuratorDb())); } private static String appDimension(Application application) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 53476a2e42f..797d2b9aa0e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -62,7 +62,6 @@ public class ControllerContainerTest { " \n" + " \n" + " \n" + - " \n" + " \n" + " \n" + " \n" + diff --git a/controller-server/src/test/resources/chef_output.json b/controller-server/src/test/resources/chef_output.json deleted file mode 100644 index 257065f7b5b..00000000000 --- a/controller-server/src/test/resources/chef_output.json +++ /dev/null @@ -1,34 +0,0 @@ -{ - "total": 1, - "start": 0, - "rows": [ - { - "url": "https://chef-server.test/organizations/vespa/nodes/fake-node.test", - "data": { - "fqdn": "fake-node.test", - "ohai_time": 1475497186.68962, - "tenant": "ciintegrationtests", - "application": "restart", - "instance": "default", - "zone": "cd_cd-us-east-1_prod", - "system": "cd", - "environment": "prod", - "region": "cd-us-east-1" - } - }, - { - "url": "https://chef-server.test/organizations/vespa/nodes/fake-node2.test", - "data": { - "fqdn": "fake-node2.test", - "ohai_time": 1475497186.68962, - "tenant": null, - "application": null, - "instance": null, - "zone": null, - "system": null, - "environment": null, - "region": null - } - } - ] -} -- cgit v1.2.3 From 5407c03b494d907d16946966cc06179f160871d0 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 3 Jul 2019 10:58:58 +0200 Subject: Remove unreferenced test files --- .../restapi/application/responses/test-config.json | 21 --------------------- .../src/test/resources/job-grandparent.json | 4 ---- .../src/test/resources/job-parent.json | 9 --------- 3 files changed, 34 deletions(-) delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json delete mode 100644 controller-server/src/test/resources/job-grandparent.json delete mode 100644 controller-server/src/test/resources/job-parent.json (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json deleted file mode 100644 index fef3cf6a372..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json +++ /dev/null @@ -1,21 +0,0 @@ -{ - "application": "tenant1:application1:default", - "zone": "dev.us-east-1", - "system": "main", - "endpoints": { - "dev.us-east-1": [ - "http://old-endpoint.vespa.yahooapis.com:4080" - ], - "prod.us-central-1": [ - "http://old-endpoint.vespa.yahooapis.com:4080" - ] - }, - "zoneEndpoints": { - "dev.us-east-1": { - "default": "http://old-endpoint.vespa.yahooapis.com:4080" - }, - "prod.us-central-1": { - "default": "http://old-endpoint.vespa.yahooapis.com:4080" - } - } -} diff --git a/controller-server/src/test/resources/job-grandparent.json b/controller-server/src/test/resources/job-grandparent.json deleted file mode 100644 index 63602bed146..00000000000 --- a/controller-server/src/test/resources/job-grandparent.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "duration": 720000, - "causes": [] -} diff --git a/controller-server/src/test/resources/job-parent.json b/controller-server/src/test/resources/job-parent.json deleted file mode 100644 index 88d50de394f..00000000000 --- a/controller-server/src/test/resources/job-parent.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "duration": 1200000, - "causes": [ - { - "upstreamBuild": 231, - "upstreamProject": "3-v3-job-grandparent" - } - ] -} -- cgit v1.2.3 From 03e0c67fbbbe13613bbf34affa8c7ad9eef32902 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 3 Jul 2019 14:52:35 +0200 Subject: Do not return empty JSON response --- .../restapi/application/ApplicationApiHandler.java | 4 ++-- .../restapi/application/EmptyJsonResponse.java | 23 ---------------------- .../restapi/application/EmptyResponse.java | 20 +++++++++++++++++++ .../restapi/deployment/DeploymentApiHandler.java | 8 ++++---- .../controller/restapi/user/UserApiHandler.java | 7 +++---- 5 files changed, 29 insertions(+), 33 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index dce7d2e68ac..702fecf78f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -267,7 +267,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } @@ -1097,7 +1097,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); - return new EmptyJsonResponse(); // TODO: Replicates current behavior but should return a message response instead + return new StringResponse("Deleted application " + id); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java deleted file mode 100644 index be3222cc1a8..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyJsonResponse.java +++ /dev/null @@ -1,23 +0,0 @@ -// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.restapi.application; - -import com.yahoo.container.jdisc.HttpResponse; - -import java.io.OutputStream; - -/** - * @author bratseth - */ -public class EmptyJsonResponse extends HttpResponse { - - public EmptyJsonResponse() { - super(200); - } - - @Override - public void render(OutputStream stream) {} - - @Override - public String getContentType() { return "application/json"; } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java new file mode 100644 index 00000000000..e343615f066 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/EmptyResponse.java @@ -0,0 +1,20 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.restapi.application; + +import com.yahoo.container.jdisc.HttpResponse; + +import java.io.OutputStream; + +/** + * @author bratseth + */ +public class EmptyResponse extends HttpResponse { + + public EmptyResponse() { + super(200); + } + + @Override + public void render(OutputStream stream) {} + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java index 978b7e4397d..44b67a186b8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/deployment/DeploymentApiHandler.java @@ -8,18 +8,18 @@ import com.yahoo.config.provision.HostName; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; +import com.yahoo.restapi.Path; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.JobList; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; import com.yahoo.vespa.hosted.controller.restapi.Uri; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; -import com.yahoo.restapi.Path; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; +import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.yolean.Exceptions; import java.util.Optional; @@ -71,7 +71,7 @@ public class DeploymentApiHandler extends LoggingRequestHandler { private HttpResponse handleOPTIONS() { // We implement this to avoid redirect loops on OPTIONS requests from browsers, but do not really bother // spelling out the methods supported at each path, which we should - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,OPTIONS"); return response; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java index 5ef997b6d55..7a76f13392d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiHandler.java @@ -13,20 +13,19 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.config.SlimeUtils; +import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.integration.user.UserId; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import com.yahoo.vespa.hosted.controller.api.integration.user.Roles; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.application.EmptyJsonResponse; +import com.yahoo.vespa.hosted.controller.restapi.application.EmptyResponse; import com.yahoo.yolean.Exceptions; import java.util.ArrayList; import java.util.Collections; -import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -99,7 +98,7 @@ public class UserApiHandler extends LoggingRequestHandler { } private HttpResponse handleOPTIONS() { - EmptyJsonResponse response = new EmptyJsonResponse(); + EmptyResponse response = new EmptyResponse(); response.headers().put("Allow", "GET,PUT,POST,PATCH,DELETE,OPTIONS"); return response; } -- cgit v1.2.3 From 37da06b45f307aa94fba6642514f6c3748fc8d5e Mon Sep 17 00:00:00 2001 From: Bjørn Christian Seime Date: Wed, 3 Jul 2019 13:43:57 +0200 Subject: Use TlsContextBasedProvider in ControllerSslContextFactoryProvider --- .../tls/ControllerSslContextFactoryProvider.java | 37 ++++++++++------------ 1 file changed, 16 insertions(+), 21 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java index d50d141d625..1ac82317695 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/tls/ControllerSslContextFactoryProvider.java @@ -2,15 +2,17 @@ package com.yahoo.vespa.hosted.controller.tls; import com.google.inject.Inject; -import com.yahoo.component.AbstractComponent; import com.yahoo.container.jdisc.secretstore.SecretStore; -import com.yahoo.jdisc.http.ssl.SslContextFactoryProvider; +import com.yahoo.jdisc.http.ssl.impl.TlsContextBasedProvider; import com.yahoo.security.KeyStoreBuilder; import com.yahoo.security.KeyStoreType; import com.yahoo.security.KeyUtils; +import com.yahoo.security.SslContextBuilder; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.security.tls.DefaultTlsContext; +import com.yahoo.security.tls.PeerAuthentication; +import com.yahoo.security.tls.TlsContext; import com.yahoo.vespa.hosted.controller.tls.config.TlsConfig; -import org.eclipse.jetty.util.ssl.SslContextFactory; import java.nio.file.Files; import java.nio.file.Paths; @@ -28,11 +30,11 @@ import java.util.concurrent.ConcurrentHashMap; * @author bjorncs */ @SuppressWarnings("unused") // Injected -public class ControllerSslContextFactoryProvider extends AbstractComponent implements SslContextFactoryProvider { +public class ControllerSslContextFactoryProvider extends TlsContextBasedProvider { private final KeyStore truststore; private final KeyStore keystore; - private final Map sslContextFactories = new ConcurrentHashMap<>(); + private final Map tlsContextMap = new ConcurrentHashMap<>(); @Inject public ControllerSslContextFactoryProvider(SecretStore secretStore, TlsConfig config) { @@ -50,24 +52,17 @@ public class ControllerSslContextFactoryProvider extends AbstractComponent imple } @Override - public SslContextFactory getInstance(String containerId, int port) { - return sslContextFactories.computeIfAbsent(port, this::createSslContextFactory); + protected TlsContext getTlsContext(String containerId, int port) { + return tlsContextMap.computeIfAbsent(port, this::createTlsContext); } - /** Create a SslContextFactory backed by an in-memory key and trust store */ - private SslContextFactory createSslContextFactory(int port) { - // TODO Use DefaultTlsContext to configure SslContextFactory (ensure that cipher/protocol configuration is same across all TLS endpoints). - - SslContextFactory.Server factory = new SslContextFactory.Server(); - if (port != 443) { - factory.setWantClientAuth(true); - } - factory.setTrustStore(truststore); - factory.setKeyStore(keystore); - factory.setKeyStorePassword(""); - factory.setExcludeProtocols("TLSv1.3"); // TLSv1.3 is broken is multiple OpenJDK 11 versions - factory.setEndpointIdentificationAlgorithm(null); // disable https hostname verification of clients (must be disabled when using Athenz x509 certificates) - return factory; + private TlsContext createTlsContext(int port) { + return new DefaultTlsContext( + new SslContextBuilder() + .withKeyStore(keystore, new char[0]) + .withTrustStore(truststore) + .build(), + port != 443 ? PeerAuthentication.WANT : PeerAuthentication.DISABLED); } /** Get private key from secret store **/ -- cgit v1.2.3 From b8aed836516a9867474b15fbe2961c99171606ef Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Wed, 3 Jul 2019 15:32:19 +0200 Subject: Better JSON responses --- .../restapi/application/ApplicationApiHandler.java | 24 ++++------------ .../restapi/application/ApplicationApiTest.java | 32 +++++++++++----------- .../controller/restapi/user/UserApiTest.java | 2 +- 3 files changed, 23 insertions(+), 35 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 702fecf78f2..9c320df2f6c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -29,9 +29,7 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.ActivateResult; -import com.yahoo.vespa.hosted.controller.api.application.v4.ApplicationResource; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.TenantResource; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; @@ -68,7 +66,6 @@ import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse; import com.yahoo.vespa.hosted.controller.restapi.SlimeJsonResponse; -import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; @@ -945,12 +942,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { Optional hostname = Optional.ofNullable(request.getProperty("hostname")).map(Hostname::new); controller.applications().restart(deploymentId, hostname); - // TODO: Change to return JSON - return new StringResponse("Requested restart of " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - EnvironmentResource.API_PATH, environment, - "region", region, - "instance", instanceName)); + return new MessageResponse("Requested restart of " + deploymentId); } private HttpResponse jobDeploy(ApplicationId id, JobType type, HttpRequest request) { @@ -1097,21 +1089,17 @@ public class ApplicationApiHandler extends LoggingRequestHandler { ? Optional.empty() : Optional.of(accessControlRequests.credentials(id.tenant(), toSlime(request.getData()).get(), request.getJDiscRequest())); controller.applications().deleteApplication(id, credentials); - return new StringResponse("Deleted application " + id); + return new MessageResponse("Deleted application " + id); } private HttpResponse deactivate(String tenantName, String applicationName, String instanceName, String environment, String region, HttpRequest request) { Application application = controller.applications().require(ApplicationId.from(tenantName, applicationName, instanceName)); // Attempt to deactivate application even if the deployment is not known by the controller - controller.applications().deactivate(application.id(), ZoneId.from(environment, region)); - - // TODO: Change to return JSON - return new StringResponse("Deactivated " + path(TenantResource.API_PATH, tenantName, - ApplicationResource.API_PATH, applicationName, - "instance", instanceName, - EnvironmentResource.API_PATH, environment, - "region", region)); + DeploymentId deploymentId = new DeploymentId(application.id(), ZoneId.from(environment, region)); + controller.applications().deactivate(deploymentId.applicationId(), deploymentId.zoneId()); + + return new MessageResponse("Deactivated " + deploymentId); } private HttpResponse notifyJobCompletion(String tenant, String application, HttpRequest request) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index a0acbc625f7..29931a1f626 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -241,7 +241,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-east-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/test/region/us-east-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in test.us-east-1\"}"); controllerTester.jobCompletion(JobType.systemTest) .application(id) @@ -255,7 +255,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-east-3/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/staging/region/us-east-3"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in staging.us-east-3\"}"); controllerTester.jobCompletion(JobType.stagingTest) .application(id) .projectId(screwdriverProjectId) @@ -367,7 +367,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant2/application/application2", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant2.application2\"}"); // Set version 6.1 to broken to change compile version for. controllerTester.upgrader().overrideConfidence(Version.fromString("6.1"), VespaVersion.Confidence.broken); @@ -480,27 +480,27 @@ public class ApplicationApiTest extends ControllerContainerTest { // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); // POST a 'restart application' command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in prod.us-central-1\"}"); // POST a 'restart application' in staging environment command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/staging/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in staging.us-central-1\"}"); // POST a 'restart application' in staging test command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1/restart", POST) .screwdriverIdentity(SCREWDRIVER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/test/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in test.us-central-1\"}"); // POST a 'restart application' in staging dev command tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1/restart", POST) .userIdentity(USER_ID), - "Requested restart of tenant/tenant1/application/application1/environment/dev/region/us-central-1/instance/instance1"); + "{\"message\":\"Requested restart of tenant1.application1.instance1 in dev.us-central-1\"}"); // POST a 'restart application' command with a host filter (other filters not supported yet) tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1/restart?hostname=host1", POST) @@ -530,18 +530,18 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE (deactivate) a deployment - dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/instance1", DELETE) .userIdentity(USER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/dev/region/us-west-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in dev.us-west-1\"}"); // DELETE (deactivate) a deployment - prod tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // DELETE (deactivate) a deployment is idempotent tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/instance1", DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-central-1"); + "{\"message\":\"Deactivated tenant1.application1.instance1 in prod.us-central-1\"}"); // POST an application package to start a deployment to dev tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1", POST) @@ -661,7 +661,7 @@ public class ApplicationApiTest extends ControllerContainerTest { // DELETE an application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant1.application1.instance1\"}"); // DELETE a tenant tester.assertResponse(request("/application/v4/tenant/tenant1", DELETE).userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), @@ -990,7 +990,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), - ""); + "{\"message\":\"Deleted application tenant1.application1.instance1\"}"); // DELETE application again - should produce 404 tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", DELETE) .oktaAccessToken(OKTA_AT) @@ -1089,7 +1089,7 @@ public class ApplicationApiTest extends ControllerContainerTest { tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1", DELETE) .userIdentity(authorizedUser) .oktaAccessToken(OKTA_AT), - "", + "{\"message\":\"Deleted application tenant1.application1\"}", 200); // Updating a tenant for an Athens domain the user is not admin for is disallowed @@ -1528,7 +1528,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request(testPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated " + testPath.replaceFirst("/application/v4/", "")); + "{\"message\":\"Deactivated " + application + " in test.us-east-1\"}"); controllerTester.jobCompletion(JobType.systemTest) .application(application) .projectId(projectId) @@ -1543,7 +1543,7 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("deploy-result.json")); tester.assertResponse(request(stagingPath, DELETE) .screwdriverIdentity(SCREWDRIVER_ID), - "Deactivated " + stagingPath.replaceFirst("/application/v4/", "")); + "{\"message\":\"Deactivated " + application + " in staging.us-east-3\"}"); controllerTester.jobCompletion(JobType.stagingTest) .application(application) .projectId(projectId) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java index 59f63f0472a..d0e9ae77965 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/user/UserApiTest.java @@ -137,7 +137,7 @@ public class UserApiTest extends ControllerContainerCloudTest { // DELETE an application is available to application admins. tester.assertResponse(request("/application/v4/tenant/my-tenant/application/my-app", DELETE) .roles(Set.of(Role.applicationAdmin(id.tenant(), id.application()))), - ""); + "{\"message\":\"Deleted application my-tenant.my-app\"}"); // DELETE a tenant role is available to tenant admins. tester.assertResponse(request("/user/v1/tenant/my-tenant", DELETE) -- cgit v1.2.3 From c5f0e6c51fc4db50b52fb52bc7b8817167840233 Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Fri, 5 Jul 2019 12:27:08 +0200 Subject: Remove constraint that we can have only one main endpoint in EndpointList --- .../yahoo/vespa/hosted/controller/application/EndpointList.java | 7 ------- 1 file changed, 7 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index feedc1c8f9d..d9aea783880 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -25,13 +25,6 @@ public class EndpointList { private final List endpoints; private EndpointList(List endpoints) { - long mainEndpoints = endpoints.stream() - .filter(endpoint -> endpoint.scope() == Endpoint.Scope.global) - .filter(Predicate.not(Endpoint::directRouting)) - .filter(Predicate.not(Endpoint::legacy)).count(); - if (mainEndpoints > 1) { - throw new IllegalArgumentException("Can have only 1 non-legacy global endpoint, got " + endpoints); - } if (endpoints.stream().distinct().count() != endpoints.size()) { throw new IllegalArgumentException("Expected all endpoints to be distinct, got " + endpoints); } -- cgit v1.2.3 From 636bdeee208c906d16532b25b5ba5d3ed53f1091 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 5 Jul 2019 15:22:03 +0200 Subject: Always show lastVersions and deploying elements --- .../restapi/application/JobControllerApiHandlerHelper.java | 14 +++++++------- .../restapi/application/responses/dev-overview.json | 2 ++ 2 files changed, 9 insertions(+), 7 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index cd7c0d6236d..b41598f021c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -6,26 +6,26 @@ import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.slime.Cursor; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.NotExistsException; +import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.JobStatus; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.SourceRevision; import com.yahoo.vespa.hosted.controller.deployment.DeploymentSteps; import com.yahoo.vespa.hosted.controller.deployment.JobController; -import com.yahoo.vespa.hosted.controller.api.integration.LogEntry; -import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Run; +import com.yahoo.vespa.hosted.controller.deployment.RunLog; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.restapi.MessageResponse; @@ -94,14 +94,14 @@ class JobControllerApiHandlerHelper { Slime slime = new Slime(); Cursor responseObject = slime.setObject(); + Cursor lastVersionsObject = responseObject.setObject("lastVersions"); if (application.deploymentJobs().statusOf(component).flatMap(JobStatus::lastSuccess).isPresent()) { - Cursor lastVersionsObject = responseObject.setObject("lastVersions"); lastPlatformToSlime(lastVersionsObject.setObject("platform"), controller, application, change, steps); lastApplicationToSlime(lastVersionsObject.setObject("application"), application, change, steps, controller); } + Cursor deployingObject = responseObject.setObject("deploying"); if ( ! change.isEmpty()) { - Cursor deployingObject = responseObject.setObject("deploying"); change.platform().ifPresent(version -> deployingObject.setString("platform", version.toString())); change.application().ifPresent(version -> applicationVersionToSlime(deployingObject.setObject("application"), version)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index c845d31a5fc..3f526c27221 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -1,4 +1,6 @@ { + "lastVersions": {}, + "deploying": {}, "deployments": [], "jobs": {}, "devJobs": { -- cgit v1.2.3 From 707db1edda0f41e91fae5e0f27611b46db9c79e0 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Fri, 5 Jul 2019 15:22:25 +0200 Subject: Put devJobs inside runs array like for jobs --- .../application/JobControllerApiHandlerHelper.java | 2 +- .../application/responses/dev-overview.json | 38 ++++++++++++---------- 2 files changed, 22 insertions(+), 18 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java index b41598f021c..b34ea79c670 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelper.java @@ -141,7 +141,7 @@ class JobControllerApiHandlerHelper { && type.environment().isManuallyDeployed() && application.deployments().containsKey(type.zone(controller.system()))) controller.jobController().last(application.id(), type) - .ifPresent(last -> runToSlime(devJobsObject.setObject(type.jobName()), + .ifPresent(last -> runToSlime(devJobsObject.setObject(type.jobName()).setArray("runs").addObject(), last, baseUriForJobs.resolve(baseUriForJobs.getPath() + "/" + type.jobName()).normalize())); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json index 3f526c27221..93b6138d987 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-overview.json @@ -5,23 +5,27 @@ "jobs": {}, "devJobs": { "dev-us-east-1": { - "id": 1, - "status": "success", - "start": 0, - "end": 0, - "wantedPlatform": "6.1", - "wantedApplication": { - "hash": "unknown" - }, - "steps": { - "deployReal": "succeeded", - "installReal": "succeeded" - }, - "tasks": { - "deploy": "succeeded", - "install": "succeeded" - }, - "log": "https://some.url:43/root/dev-us-east-1/run/1" + "runs": [ + { + "id": 1, + "status": "success", + "start": 0, + "end": 0, + "wantedPlatform": "6.1", + "wantedApplication": { + "hash": "unknown" + }, + "steps": { + "deployReal": "succeeded", + "installReal": "succeeded" + }, + "tasks": { + "deploy": "succeeded", + "install": "succeeded" + }, + "log": "https://some.url:43/root/dev-us-east-1/run/1" + } + ] } } } -- cgit v1.2.3 From e7f8a859652089e6774fbc9443ec1f16252eb68f Mon Sep 17 00:00:00 2001 From: Øyvind Grønnesby Date: Mon, 8 Jul 2019 16:53:04 +0200 Subject: Default to all declared prod zones for endpoints without regions - Extract all production zones in the DeploymentSpec constructor and update all Endpoints instances without regions to these zones - Add more tests to validate that this does what we want it to. --- .../config/application/api/DeploymentSpec.java | 27 ++++++++++++++++++- .../com/yahoo/config/application/api/Endpoint.java | 3 +++ .../config/application/api/DeploymentSpecTest.java | 31 ++++++++++++++++++++++ .../vespa/hosted/controller/ControllerTest.java | 15 ++++++++--- 4 files changed, 72 insertions(+), 4 deletions(-) (limited to 'controller-server') diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java index ada5ee23a7c..5b5f89fd8d1 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java @@ -74,11 +74,36 @@ public class DeploymentSpec { this.athenzDomain = athenzDomain; this.athenzService = athenzService; this.notifications = notifications; - this.endpoints = List.copyOf(Objects.requireNonNull(endpoints, "Missing endpoints parameter")); + this.endpoints = ImmutableList.copyOf(validateEndpoints(endpoints, this.steps)); validateZones(this.steps); validateAthenz(); validateEndpoints(this.steps, globalServiceId, this.endpoints); } + + /** Validates the endpoints and makes sure default values are respected */ + private List validateEndpoints(List endpoints, List steps) { + Objects.requireNonNull(endpoints, "Missing endpoints parameter"); + + var productionRegions = steps.stream() + .filter(step -> step.deploysTo(Environment.prod)) + .flatMap(step -> step.zones().stream()) + .flatMap(zone -> zone.region().stream()) + .map(RegionName::value) + .collect(Collectors.toSet()); + + var rebuiltEndpointsList = new ArrayList(); + + for (var endpoint : endpoints) { + if (endpoint.regions().isEmpty()) { + var rebuiltEndpoint = endpoint.withRegions(productionRegions); + rebuiltEndpointsList.add(rebuiltEndpoint); + } else { + rebuiltEndpointsList.add(endpoint); + } + } + + return ImmutableList.copyOf(rebuiltEndpointsList); + } /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */ private void validateTotalDelay(List steps) { diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java index 158fbfb175f..e47dcd78219 100644 --- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java +++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java @@ -75,4 +75,7 @@ public class Endpoint { return Objects.hash(endpointId, containerId, regions); } + public Endpoint withRegions(Set regions) { + return new Endpoint(endpointId, containerId, regions); + } } diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java index 8120c82e8f4..47eaf7a515a 100644 --- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java +++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java @@ -14,6 +14,7 @@ import java.util.List; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.config.application.api.Notifications.Role.author; import static com.yahoo.config.application.api.Notifications.When.failing; @@ -516,6 +517,28 @@ public class DeploymentSpecTest { assertEquals(List.of("fooooooooooo"), endpointIds("")); } + @Test + public void endpointDefaultRegions() { + var spec = DeploymentSpec.fromXml("" + + "" + + " " + + " us-east" + + " us-west" + + " " + + " " + + " " + + " us-east" + + " " + + " " + + " " + + " " + + ""); + + assertEquals(Set.of("us-east"), endpointRegions("foo", spec)); + assertEquals(Set.of("us-east", "us-west"), endpointRegions("nalle", spec)); + assertEquals(Set.of("us-east", "us-west"), endpointRegions("default", spec)); + } + private static void assertInvalid(String endpointTag) { try { endpointIds(endpointTag); @@ -523,6 +546,14 @@ public class DeploymentSpecTest { } catch (IllegalArgumentException ignored) {} } + private static Set endpointRegions(String endpointId, DeploymentSpec spec) { + return spec.endpoints().stream() + .filter(endpoint -> endpoint.endpointId().equals(endpointId)) + .flatMap(endpoint -> endpoint.regions().stream()) + .map(RegionName::value) + .collect(Collectors.toSet()); + } + private static List endpointIds(String endpointTag) { var xml = "" + " " + diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index c26f1879f6a..0cf54db4d3a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -43,6 +43,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; @@ -358,6 +359,7 @@ public class ControllerTest { .environment(Environment.prod) .endpoint("foobar", "qrs", "us-west-1", "us-central-1") .endpoint("default", "qrs", "us-west-1", "us-central-1") + .endpoint("all", "qrs") .region("us-west-1") .region("us-central-1") .build(); @@ -370,24 +372,31 @@ public class ControllerTest { Set.of( "rotation-id-01", "rotation-id-02", + "rotation-id-03", "app1--tenant1.global.vespa.oath.cloud", - "foobar--app1--tenant1.global.vespa.oath.cloud" + "foobar--app1--tenant1.global.vespa.oath.cloud", + "all--app1--tenant1.global.vespa.oath.cloud" ), tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone()))); } tester.flushDnsRequests(); - assertEquals(2, tester.controllerTester().nameService().records().size()); + assertEquals(3, tester.controllerTester().nameService().records().size()); var record1 = tester.controllerTester().findCname("app1--tenant1.global.vespa.oath.cloud"); assertTrue(record1.isPresent()); assertEquals("app1--tenant1.global.vespa.oath.cloud", record1.get().name().asString()); - assertEquals("rotation-fqdn-02.", record1.get().data().asString()); + assertEquals("rotation-fqdn-03.", record1.get().data().asString()); var record2 = tester.controllerTester().findCname("foobar--app1--tenant1.global.vespa.oath.cloud"); assertTrue(record2.isPresent()); assertEquals("foobar--app1--tenant1.global.vespa.oath.cloud", record2.get().name().asString()); assertEquals("rotation-fqdn-01.", record2.get().data().asString()); + + var record3 = tester.controllerTester().findCname("all--app1--tenant1.global.vespa.oath.cloud"); + assertTrue(record3.isPresent()); + assertEquals("all--app1--tenant1.global.vespa.oath.cloud", record3.get().name().asString()); + assertEquals("rotation-fqdn-02.", record3.get().data().asString()); } @Test -- cgit v1.2.3 From 37c356e4bf1c323674323e02f5a4a8ab841f250e Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 9 Jul 2019 13:00:59 +0200 Subject: Add rest endpoint for metering data (not included in the application endpoint yet --- .../hosted/controller/api/role/PathGroup.java | 3 +- .../restapi/application/ApplicationApiHandler.java | 40 ++++++++++++++++++++++ .../restapi/application/ApplicationApiTest.java | 12 ++++++- .../responses/application1-metering.json | 34 ++++++++++++++++++ 4 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-metering.json (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java index 1ec75e0c998..08702027264 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/role/PathGroup.java @@ -76,7 +76,8 @@ enum PathGroup { "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended", "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/service/{*}", - "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/{*}"), + "/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/global-rotation/{*}", + "/application/v4/tenant/{tenant}/application/{application}/metering"), /** Path used to restart development nodes. */ developmentRestart(Matcher.tenant, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9c320df2f6c..48cef8c5220 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -176,6 +176,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { if (path.matches("/application/v4/tenant/{tenant}/application/{application}")) return application(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying")) return deploying(path.get("tenant"), path.get("application"), "default", request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), "default", request); + if (path.matches("/application/v4/tenant/{tenant}/application/{application}/metering")) return metering(path.get("tenant"), path.get("application"), request); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/nodes")) return nodes(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region")); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap()); if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance")) return applications(path.get("tenant"), Optional.of(path.get("application")), request); @@ -772,6 +773,45 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } + HttpResponse metering(String tenant, String application, HttpRequest request) { + Slime slime = new Slime(); + Cursor root = slime.setObject(); + + Cursor currentRate = root.setObject("currentrate"); + currentRate.setDouble("cpu", 0); + currentRate.setDouble("mem", 0); + currentRate.setDouble("disk", 0); + + Cursor thismonth = root.setObject("thismonth"); + thismonth.setDouble("cpu", 0); + thismonth.setDouble("mem", 0); + thismonth.setDouble("disk", 0); + + Cursor lastmonth = root.setObject("lastmonth"); + lastmonth.setDouble("cpu", 0); + lastmonth.setDouble("mem", 0); + lastmonth.setDouble("disk", 0); + + Cursor details = root.setObject("details"); + + Cursor detailsCpu = details.setObject("cpu"); + Cursor detailsCpuDummyApp = detailsCpu.setObject("dummy"); + Cursor detailsCpuDummyData = detailsCpuDummyApp.setArray("data"); + + // The data array should be filled with objects like: { unixms: , valur: Date: Tue, 9 Jul 2019 13:21:22 +0200 Subject: Tester sets up the user as an operator --- .../vespa/hosted/controller/restapi/application/ApplicationApiTest.java | 1 - 1 file changed, 1 deletion(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 6db47feec22..a8f150920db 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -860,7 +860,6 @@ public class ApplicationApiTest extends ControllerContainerTest { @Test public void testMeteringResponses() { - // TODO - why does this request (for a tenant and application that does not exist) go through? tester.assertResponse(request("/application/v4/tenant/doesnotexist/application/doesnotexist/metering", GET) .userIdentity(USER_ID) .oktaAccessToken(OKTA_AT), -- cgit v1.2.3 From 1c2d659bee05d86f2ffd2a90e4748d16f8fa71fb Mon Sep 17 00:00:00 2001 From: toby Date: Tue, 9 Jul 2019 14:03:12 +0200 Subject: Make metering method private --- .../hosted/controller/restapi/application/ApplicationApiHandler.java | 2 +- .../vespa/hosted/controller/restapi/application/ApplicationApiTest.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 48cef8c5220..868272e5051 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -773,7 +773,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { return new SlimeJsonResponse(slime); } - HttpResponse metering(String tenant, String application, HttpRequest request) { + private HttpResponse metering(String tenant, String application, HttpRequest request) { Slime slime = new Slime(); Cursor root = slime.setObject(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index a8f150920db..577b8491bd2 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -857,7 +857,6 @@ public class ApplicationApiTest extends ControllerContainerTest { new File("application-without-change-multiple-deployments.json")); } - @Test public void testMeteringResponses() { tester.assertResponse(request("/application/v4/tenant/doesnotexist/application/doesnotexist/metering", GET) -- cgit v1.2.3 From 8c4858b1ecff262a71083e5e083f341784599dd8 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 9 Jul 2019 09:04:34 +0200 Subject: Combine NodeRepositoryClientInterface and NodeRepository --- .../integration/configserver/NodeRepository.java | 80 +++++++++++++++++++++- .../NodeRepositoryClientInterface.java | 58 ---------------- .../maintenance/ClusterInfoMaintainer.java | 13 ++-- .../maintenance/ControllerMaintenance.java | 10 +-- .../maintenance/CostReportMaintainer.java | 6 +- .../maintenance/ResourceMeterMaintainer.java | 19 +++-- .../controller/restapi/cost/CostApiHandler.java | 6 +- .../controller/restapi/cost/CostCalculator.java | 7 +- .../integration/NodeRepositoryClientMock.java | 54 ++++----------- .../controller/integration/NodeRepositoryMock.java | 36 +++++++++- 10 files changed, 155 insertions(+), 134 deletions(-) delete mode 100644 controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java (limited to 'controller-server') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java index 95544c23db5..916a388692e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/NodeRepository.java @@ -5,8 +5,13 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.NodeType; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import java.util.Collection; import java.util.List; +import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; @@ -17,8 +22,38 @@ import java.util.stream.Collectors; */ public interface NodeRepository { + void addNodes(ZoneId zone, Collection nodes); + + void deleteNode(ZoneId zone, String hostname); + + void setState(ZoneId zone, NodeState nodeState, String nodename); + + NodeRepositoryNode getNode(ZoneId zone, String hostname); + + NodeList listNodes(ZoneId zone); + + NodeList listNodes(ZoneId zone, ApplicationId application); + /** List all nodes in zone owned by given application */ - List list(ZoneId zone, ApplicationId application); + default List list(ZoneId zone, ApplicationId application) { + return listNodes(zone, application).nodes().stream() + .map(n -> new Node(com.yahoo.config.provision.HostName.from(n.getHostname()), + fromJacksonState(n.getState()), + fromJacksonType(n.getType()), Optional.of(application), + Version.fromString(n.getVespaVersion()), + Version.fromString(n.getWantedVespaVersion()), + Version.fromString(n.getCurrentOsVersion()), + Version.fromString(n.getWantedOsVersion()), + fromBoolean(n.getAllowedToBeDown()), + n.getCurrentRestartGeneration(), + n.getRestartGeneration(), + n.getCurrentRebootGeneration(), + n.getRebootGeneration(), + n.getCanonicalFlavor(), + n.getMembership().clusterid, + clusterTypeOf(n.getMembership().clustertype))) + .collect(Collectors.toUnmodifiableList()); + } /** List all nodes in states, in zone owned by given application */ default List list(ZoneId zone, ApplicationId application, Set states) { @@ -39,4 +74,47 @@ public interface NodeRepository { /** Cancels firmware checks on all hosts in the given zone. */ void cancelFirmwareCheck(ZoneId zone); + + + private Node.ClusterType clusterTypeOf(String type) { + switch (type) { + case "admin": return Node.ClusterType.admin; + case "content": return Node.ClusterType.content; + case "container": return Node.ClusterType.container; + default: throw new IllegalArgumentException("Unknown cluster type '" + type + "'."); + } + } + + // Convert Jackson type to config.provision type + private static NodeType fromJacksonType(com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeType nodeType) { + switch (nodeType) { + case tenant: return NodeType.tenant; + case host: return NodeType.host; + case proxy: return NodeType.proxy; + case proxyhost: return NodeType.proxyhost; + case config: return NodeType.config; + case confighost: return NodeType.confighost; + default: throw new IllegalArgumentException("Unknown type: " + nodeType); + } + } + + private static com.yahoo.vespa.hosted.controller.api.integration.configserver.Node.State fromJacksonState(NodeState state) { + switch (state) { + case provisioned: return Node.State.provisioned; + case ready: return Node.State.ready; + case reserved: return Node.State.reserved; + case active: return Node.State.active; + case inactive: return Node.State.inactive; + case dirty: return Node.State.dirty; + case failed: return Node.State.failed; + case parked: return Node.State.parked; + default: throw new IllegalArgumentException("Unknown state: " + state); + } + } + + private static Node.ServiceState fromBoolean(Boolean allowedDown) { + return (allowedDown == null) + ? Node.ServiceState.unorchestrated + : allowedDown ? Node.ServiceState.allowedDown : Node.ServiceState.expectedUp; + } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java deleted file mode 100644 index 4b495ebf331..00000000000 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/noderepository/NodeRepositoryClientInterface.java +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.api.integration.noderepository; - -import com.yahoo.config.provision.zone.ZoneId; - -import java.io.IOException; -import java.util.Collection; - -/** - * A complete client for the node repository REST API. - * - * @author smorgrav - * @author bjorncs - */ -// TODO: Get rid of all the checked exceptions -// TODO: Replace remaining controller-server usages of this with -// com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository and move this package back to internal -// repo -public interface NodeRepositoryClientInterface { - - enum WantTo { - Retire, - Deprovision - } - - void addNodes(ZoneId zone, Collection nodes) throws IOException; - - NodeRepositoryNode getNode(ZoneId zone, String hostname) throws IOException; - - void deleteNode(ZoneId zone, String hostname) throws IOException; - - NodeList listNodes(ZoneId zone, boolean recursive) throws IOException; - - NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) throws IOException; - - String resetFailureInformation(ZoneId zone, String nodename) throws IOException; - - String restart(ZoneId zone, String nodename) throws IOException; - - String reboot(ZoneId zone, String nodename) throws IOException; - - String cancelReboot(ZoneId zone, String nodename) throws IOException; - - String wantTo(ZoneId zone, String nodename, WantTo... actions) throws IOException; - - String cancelRestart(ZoneId zone, String nodename) throws IOException; - - String setHardwareFailureDescription(ZoneId zone, String nodename, String hardwareFailureDescription) throws IOException; - - void setState(ZoneId zone, NodeState nodeState, String nodename) throws IOException; - - String enableMaintenanceJob(ZoneId zone, String jobName) throws IOException; - - String disableMaintenanceJob(ZoneId zone, String jobName) throws IOException; - - MaintenanceJobList listMaintenanceJobs(ZoneId zone) throws IOException; - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java index c99e4f5951b..d9aaef6ef3b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ClusterInfoMaintainer.java @@ -5,8 +5,8 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; @@ -31,13 +31,13 @@ public class ClusterInfoMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(ClusterInfoMaintainer.class.getName()); private final Controller controller; - private final NodeRepositoryClientInterface nodeRepositoryClient; + private final NodeRepository nodeRepository; ClusterInfoMaintainer(Controller controller, Duration duration, JobControl jobControl, - NodeRepositoryClientInterface nodeRepositoryClient) { + NodeRepository nodeRepository) { super(controller, duration, jobControl); this.controller = controller; - this.nodeRepositoryClient = nodeRepositoryClient; + this.nodeRepository = nodeRepository; } private static String clusterId(NodeRepositoryNode node) { @@ -81,10 +81,7 @@ public class ClusterInfoMaintainer extends Maintainer { for (Deployment deployment : application.deployments().values()) { DeploymentId deploymentId = new DeploymentId(application.id(), deployment.zone()); try { - NodeList nodes = nodeRepositoryClient.listNodes(deploymentId.zoneId(), - deploymentId.applicationId().tenant().value(), - deploymentId.applicationId().application().value(), - deploymentId.applicationId().instance().value()); + NodeList nodes = nodeRepository.listNodes(deploymentId.zoneId(), deploymentId.applicationId()); Map clusterInfo = getClusterInfo(nodes); controller().applications().lockIfPresent(application.id(), lockedApplication -> controller.applications().store(lockedApplication.withClusterInfo(deployment.zone(), clusterInfo))); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index f34c24c497a..116ea532a11 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -5,8 +5,8 @@ import com.yahoo.component.AbstractComponent; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.organization.Billing; import com.yahoo.vespa.hosted.controller.api.integration.organization.ContactRetriever; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentIssues; @@ -60,7 +60,7 @@ public class ControllerMaintenance extends AbstractComponent { public ControllerMaintenance(MaintainerConfig maintainerConfig, ApiAuthorityConfig apiAuthorityConfig, Controller controller, CuratorDb curator, JobControl jobControl, Metric metric, DeploymentIssues deploymentIssues, OwnershipIssues ownershipIssues, - NameService nameService, NodeRepositoryClientInterface nodeRepositoryClient, + NameService nameService, NodeRepository nodeRepository, ContactRetriever contactRetriever, CostReportConsumer reportConsumer, ResourceSnapshotConsumer resourceSnapshotConsumer, @@ -75,7 +75,7 @@ public class ControllerMaintenance extends AbstractComponent { versionStatusUpdater = new VersionStatusUpdater(controller, Duration.ofMinutes(1), jobControl); upgrader = new Upgrader(controller, maintenanceInterval, jobControl, curator); readyJobsTrigger = new ReadyJobsTrigger(controller, Duration.ofMinutes(1), jobControl); - clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepositoryClient); + clusterInfoMaintainer = new ClusterInfoMaintainer(controller, Duration.ofHours(2), jobControl, nodeRepository); clusterUtilizationMaintainer = new ClusterUtilizationMaintainer(controller, Duration.ofHours(2), jobControl); deploymentMetricsMaintainer = new DeploymentMetricsMaintainer(controller, Duration.ofMinutes(5), jobControl); applicationOwnershipConfirmer = new ApplicationOwnershipConfirmer(controller, Duration.ofHours(12), jobControl, ownershipIssues); @@ -84,8 +84,8 @@ public class ControllerMaintenance extends AbstractComponent { osUpgraders = osUpgraders(controller, jobControl); osVersionStatusUpdater = new OsVersionStatusUpdater(controller, maintenanceInterval, jobControl); contactInformationMaintainer = new ContactInformationMaintainer(controller, Duration.ofHours(12), jobControl, contactRetriever); - costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepositoryClient, Clock.systemUTC(), selfHostedCostConfig); - resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepositoryClient, Clock.systemUTC(), metric, resourceSnapshotConsumer); + costReportMaintainer = new CostReportMaintainer(controller, Duration.ofHours(2), reportConsumer, jobControl, nodeRepository, Clock.systemUTC(), selfHostedCostConfig); + resourceMeterMaintainer = new ResourceMeterMaintainer(controller, Duration.ofMinutes(60), jobControl, nodeRepository, Clock.systemUTC(), metric, resourceSnapshotConsumer); nameServiceDispatcher = new NameServiceDispatcher(controller, Duration.ofSeconds(10), jobControl, nameService); billingMaintainer = new BillingMaintainer(controller, Duration.ofDays(3), jobControl, billing); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java index 30bca180c0f..f3dac2be22e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CostReportMaintainer.java @@ -5,7 +5,7 @@ import com.google.inject.Inject; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.restapi.cost.CostCalculator; import com.yahoo.vespa.hosted.controller.restapi.cost.CostReportConsumer; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; @@ -27,7 +27,7 @@ public class CostReportMaintainer extends Maintainer { private static final Logger log = Logger.getLogger(CostReportMaintainer.class.getName()); private final CostReportConsumer consumer; - private final NodeRepositoryClientInterface nodeRepository; + private final NodeRepository nodeRepository; private final Clock clock; private final SelfHostedCostConfig selfHostedCostConfig; @@ -36,7 +36,7 @@ public class CostReportMaintainer extends Maintainer { public CostReportMaintainer(Controller controller, Duration interval, CostReportConsumer consumer, JobControl jobControl, - NodeRepositoryClientInterface nodeRepository, + NodeRepository nodeRepository, Clock clock, SelfHostedCostConfig selfHostedCostConfig) { super(controller, interval, jobControl, "CostReportMaintainer", EnumSet.of(SystemName.main)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index c4f0597572b..a6df146fa7a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -1,29 +1,28 @@ // Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.maintenance; +import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.jdisc.Metric; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshot; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceSnapshotConsumer; -import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.*; +import java.util.Collections; +import java.util.List; +import java.util.Map; import java.util.function.Predicate; import java.util.stream.Collectors; -import static com.yahoo.yolean.Exceptions.uncheck; - /** * Creates a ResourceSnapshot per application, which is then passed on to a ResourceSnapshotConsumer * TODO: Write JSON blob of node repo somewhere @@ -34,7 +33,7 @@ public class ResourceMeterMaintainer extends Maintainer { private final Clock clock; private final Metric metric; - private final NodeRepositoryClientInterface nodeRepository; + private final NodeRepository nodeRepository; private final ResourceSnapshotConsumer resourceSnapshotConsumer; private static final String metering_last_reported = "metering_last_reported"; @@ -44,7 +43,7 @@ public class ResourceMeterMaintainer extends Maintainer { public ResourceMeterMaintainer(Controller controller, Duration interval, JobControl jobControl, - NodeRepositoryClientInterface nodeRepository, + NodeRepository nodeRepository, Clock clock, Metric metric, ResourceSnapshotConsumer resourceSnapshotConsumer) { @@ -83,7 +82,7 @@ public class ResourceMeterMaintainer extends Maintainer { return controller().zoneRegistry().zones() .ofCloud(CloudName.from("aws")) .reachable().zones().stream() - .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId(), true).nodes().stream())) + .flatMap(zone -> nodeRepository.listNodes(zone.getId()).nodes().stream()) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .filter(node -> node.getState() == NodeState.active) .collect(Collectors.toList()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java index bae790a49ad..796f786d823 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostApiHandler.java @@ -6,7 +6,7 @@ import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; import com.yahoo.restapi.Path; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse; import com.yahoo.vespa.hosted.controller.restapi.StringResponse; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; @@ -19,10 +19,10 @@ import static com.yahoo.jdisc.http.HttpRequest.Method.GET; public class CostApiHandler extends LoggingRequestHandler { private final Controller controller; - private final NodeRepositoryClientInterface nodeRepository; + private final NodeRepository nodeRepository; private final SelfHostedCostConfig selfHostedCostConfig; - public CostApiHandler(Context ctx, Controller controller, NodeRepositoryClientInterface nodeRepository, SelfHostedCostConfig selfHostedCostConfig) { + public CostApiHandler(Context ctx, Controller controller, NodeRepository nodeRepository, SelfHostedCostConfig selfHostedCostConfig) { super(ctx); this.controller = controller; this.nodeRepository = nodeRepository; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java index c44a80f7a20..919cade1b05 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/cost/CostCalculator.java @@ -2,11 +2,10 @@ package com.yahoo.vespa.hosted.controller.restapi.cost; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.resource.ResourceAllocation; import com.yahoo.vespa.hosted.controller.restapi.cost.config.SelfHostedCostConfig; @@ -26,7 +25,7 @@ public class CostCalculator { private static final double SELF_HOSTED_DISCOUNT = .5; - public static String resourceShareByPropertyToCsv(NodeRepositoryClientInterface nodeRepository, + public static String resourceShareByPropertyToCsv(NodeRepository nodeRepository, Controller controller, Clock clock, SelfHostedCostConfig selfHostedCostConfig, @@ -36,7 +35,7 @@ public class CostCalculator { List nodes = controller.zoneRegistry().zones() .reachable().in(Environment.prod).ofCloud(cloudName).zones().stream() - .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId(), true).nodes().stream())) + .flatMap(zone -> uncheck(() -> nodeRepository.listNodes(zone.getId()).nodes().stream())) .filter(node -> node.getOwner() != null && !node.getOwner().getTenant().equals("hosted-vespa")) .collect(Collectors.toList()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java index 1c7abd2489d..4f2d85adfbe 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryClientMock.java @@ -1,14 +1,16 @@ // Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.integration; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.MaintenanceJobList; +import com.yahoo.component.Version; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeMembership; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeOwner; -import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryClientInterface; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; -import com.yahoo.config.provision.zone.ZoneId; import java.util.Collection; import java.util.List; @@ -16,7 +18,7 @@ import java.util.List; /** * @author bjorncs */ -public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { +public class NodeRepositoryClientMock implements NodeRepository { @Override public void addNodes(ZoneId zone, Collection nodes) { @@ -34,14 +36,14 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { } @Override - public NodeList listNodes(ZoneId zone, boolean recursive) { + public NodeList listNodes(ZoneId zone) { NodeRepositoryNode nodeA = createNodeA(); NodeRepositoryNode nodeB = createNodeB(); return new NodeList(List.of(nodeA, nodeB)); } @Override - public NodeList listNodes(ZoneId zone, String tenant, String applicationId, String instance) { + public NodeList listNodes(ZoneId zone, ApplicationId application) { NodeRepositoryNode nodeA = createNodeA(); NodeRepositoryNode nodeB = createNodeB(); return new NodeList(List.of(nodeA, nodeB)); @@ -90,57 +92,27 @@ public class NodeRepositoryClientMock implements NodeRepositoryClientInterface { } @Override - public String resetFailureInformation(ZoneId zone, String nodename) { - throw new UnsupportedOperationException(); - } - - @Override - public String restart(ZoneId zone, String nodename) { - throw new UnsupportedOperationException(); - } - - @Override - public String reboot(ZoneId zone, String nodename) { - throw new UnsupportedOperationException(); - } - - @Override - public String cancelReboot(ZoneId zone, String nodename) { - throw new UnsupportedOperationException(); - } - - @Override - public String wantTo(ZoneId zone, String nodename, WantTo... actions) { - throw new UnsupportedOperationException(); - } - - @Override - public String cancelRestart(ZoneId zone, String nodename) { - throw new UnsupportedOperationException(); - } - - @Override - public String setHardwareFailureDescription(ZoneId zone, String nodename, String hardwareFailureDescription) { + public void setState(ZoneId zone, NodeState nodeState, String nodename) { throw new UnsupportedOperationException(); } @Override - public void setState(ZoneId zone, NodeState nodeState, String nodename) { + public void upgrade(ZoneId zone, NodeType type, Version version) { throw new UnsupportedOperationException(); } @Override - public String enableMaintenanceJob(ZoneId zone, String jobName) { + public void upgradeOs(ZoneId zone, NodeType type, Version version) { throw new UnsupportedOperationException(); } @Override - public String disableMaintenanceJob(ZoneId zone, String jobName) { + public void requestFirmwareCheck(ZoneId zone) { throw new UnsupportedOperationException(); } @Override - public MaintenanceJobList listMaintenanceJobs(ZoneId zone) { + public void cancelFirmwareCheck(ZoneId zone) { throw new UnsupportedOperationException(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java index 49bc910ac33..45c2df5cb77 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/NodeRepositoryMock.java @@ -5,11 +5,15 @@ import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeType; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeRepository; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeList; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeRepositoryNode; +import com.yahoo.vespa.hosted.controller.api.integration.noderepository.NodeState; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -55,6 +59,36 @@ public class NodeRepositoryMock implements NodeRepository { .orElseThrow(() -> new NoSuchElementException("No node with the hostname " + hostName + " is known.")); } + @Override + public void addNodes(ZoneId zone, Collection nodes) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteNode(ZoneId zone, String hostname) { + throw new UnsupportedOperationException(); + } + + @Override + public void setState(ZoneId zone, NodeState nodeState, String nodename) { + throw new UnsupportedOperationException(); + } + + @Override + public NodeRepositoryNode getNode(ZoneId zone, String hostname) { + throw new UnsupportedOperationException(); + } + + @Override + public NodeList listNodes(ZoneId zone) { + throw new UnsupportedOperationException(); + } + + @Override + public NodeList listNodes(ZoneId zone, ApplicationId application) { + throw new UnsupportedOperationException(); + } + @Override public List list(ZoneId zone, ApplicationId application) { return nodeRepository.getOrDefault(zone, Collections.emptyMap()).values().stream() -- cgit v1.2.3 From 7f7c424f5795e622c5afd064989d48fcb9c85160 Mon Sep 17 00:00:00 2001 From: Valerij Fredriksen Date: Tue, 9 Jul 2019 09:04:50 +0200 Subject: Simplify method --- .../controller/maintenance/ResourceMeterMaintainer.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java index a6df146fa7a..5e2ba48da3f 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ResourceMeterMaintainer.java @@ -89,17 +89,10 @@ public class ResourceMeterMaintainer extends Maintainer { } private Map getResourceAllocationByApplication(List nodes) { - Map> applicationNodes = new HashMap<>(); - - nodes.stream().forEach(node -> applicationNodes.computeIfAbsent(applicationIdFromNodeOwner(node.getOwner()), n -> new ArrayList<>()).add(node)); - - return applicationNodes.entrySet().stream() - .collect( - Collectors.toMap( - entry -> entry.getKey(), - entry -> ResourceAllocation.from(entry.getValue()) - ) - ); + return nodes.stream() + .collect(Collectors.groupingBy( + node -> applicationIdFromNodeOwner(node.getOwner()), + Collectors.collectingAndThen(Collectors.toList(), ResourceAllocation::from))); } private ApplicationId applicationIdFromNodeOwner(NodeOwner owner) { -- cgit v1.2.3 From 0e896cf9b16bae90ed684099ba576b14860ef591 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Wed, 10 Jul 2019 15:28:30 +0200 Subject: Remove unused feature flag --- .../vespa/hosted/controller/ApplicationController.java | 13 +------------ .../com/yahoo/vespa/hosted/controller/ControllerTest.java | 2 -- flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 6 ------ 3 files changed, 1 insertion(+), 20 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index d015e65a5e1..60fd095eb04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -2,7 +2,6 @@ package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; -import com.yahoo.collections.ArraySet; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -89,7 +88,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -130,7 +128,6 @@ public class ApplicationController { private final RoutingGenerator routingGenerator; private final RoutingPolicies routingPolicies; private final Clock clock; - private final BooleanFlag redirectLegacyDnsFlag; private final BooleanFlag useMultipleEndpoints; private final DeploymentTrigger deploymentTrigger; private final BooleanFlag provisionApplicationCertificate; @@ -148,7 +145,6 @@ public class ApplicationController { this.routingGenerator = routingGenerator; this.routingPolicies = new RoutingPolicies(controller); this.clock = clock; - this.redirectLegacyDnsFlag = Flags.REDIRECT_LEGACY_DNS_NAMES.bindTo(controller.flagSource()); this.useMultipleEndpoints = Flags.MULTIPLE_GLOBAL_ENDPOINTS.bindTo(controller.flagSource()); this.artifactRepository = artifactRepository; @@ -485,20 +481,13 @@ public class ApplicationController { application = application.with(createDefaultGlobalIdRotation(application.get(), rotation)); store(application); // store assigned rotation even if deployment fails - boolean redirectLegacyDns = redirectLegacyDnsFlag.with(FetchVector.Dimension.APPLICATION_ID, application.get().id().serializedForm()) - .value(); - EndpointList globalEndpoints = application.get() .endpointsIn(controller.system()) .scope(Endpoint.Scope.global); globalEndpoints.main().ifPresent(mainEndpoint -> { registerCname(mainEndpoint.dnsName(), rotation.name()); - if (redirectLegacyDns) { - globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), mainEndpoint.dnsName())); - } else { - globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); - } + globalEndpoints.legacy(true).asList().forEach(endpoint -> registerCname(endpoint.dnsName(), rotation.name())); }); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 0cf54db4d3a..17ba19d8f7d 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -7,7 +7,6 @@ import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.application.api.ValidationOverrides; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ApplicationName; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; @@ -43,7 +42,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.function.BiConsumer; import java.util.function.Function; import java.util.function.Supplier; diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 872b2e6a10f..0981d7b84e2 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -128,12 +128,6 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); - public static final UnboundBooleanFlag REDIRECT_LEGACY_DNS_NAMES = defineFeatureFlag( - "redirect-legacy-dns", false, - "Redirect legacy DNS names to the main DNS name", - "Takes effect on deployment through controller", - APPLICATION_ID); - public static final UnboundBooleanFlag CONFIG_SERVER_FAIL_IF_ACTIVE_SESSION_CANNOT_BE_LOADED = defineFeatureFlag( "config-server-fail-if-active-session-cannot-be-loaded", false, "Whether to fail or just log if loading an active session fails at startup of config server", -- cgit v1.2.3 From fb2e54e7d204e892d6623889e8521b04b74b76df Mon Sep 17 00:00:00 2001 From: gjoranv Date: Thu, 11 Jul 2019 12:32:51 +0200 Subject: Replace 'jdisc' with 'container' in controller-server --- .../vespa/hosted/controller/restapi/ControllerContainerTest.java | 4 ++-- .../hosted/controller/restapi/statuspage/StatusPageProxyApiTest.java | 4 ++-- .../com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) (limited to 'controller-server') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java index 797d2b9aa0e..11aa132b478 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ControllerContainerTest.java @@ -43,7 +43,7 @@ public class ControllerContainerTest { public void stopContainer() { container.close(); } private String controllerServicesXml() { - return "\n" + + return "\n" + " \n" + " 10\n" + " \n" + @@ -113,7 +113,7 @@ public class ControllerContainerTest { " http://*/zone/v2/*\n" + " \n" + variablePartXml() + - ""; + ""; } protected SystemName system() { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyApiTest.java index 2d4c43f1d44..f39914fd8fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/statuspage/StatusPageProxyApiTest.java @@ -76,7 +76,7 @@ public class StatusPageProxyApiTest { private String servicesXml() { String statusPageApiUrl = "http://127.0.0.1:" + wireMock.port(); - return "\n" + + return "\n" + " \n" + " " + statusPageApiUrl + "\n" + " \n" + @@ -87,7 +87,7 @@ public class StatusPageProxyApiTest { " \n" + " \n" + " \n" + - ""; + ""; } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java index c2c8f592738..c1188778292 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/tls/SecureContainerTest.java @@ -59,7 +59,7 @@ public class SecureContainerTest { } private String servicesXml(Path trustStore) { - return "\n" + + return "\n" + " \n" + " 10\n" + " \n" + @@ -74,7 +74,7 @@ public class SecureContainerTest { " \n" + " \n" + " \n" + - ""; + ""; } private Path writeKeyStore() { -- cgit v1.2.3