From 164676ddc2f18bc30c656b817975adf8a720aab0 Mon Sep 17 00:00:00 2001 From: Jon Marius Venstad Date: Thu, 18 Nov 2021 12:40:13 +0100 Subject: We always use osgi-based teest runner now --- .../deployment/InternalStepRunnerTest.java | 12 ++----- .../src/test/resources/test_runner_services.xml-cd | 38 ++++++++++++++++++++++ .../resources/test_runner_services.xml-cd-legacy | 22 ------------- .../resources/test_runner_services.xml-cd-osgi | 38 ---------------------- 4 files changed, 41 insertions(+), 69 deletions(-) create mode 100644 controller-server/src/test/resources/test_runner_services.xml-cd delete mode 100644 controller-server/src/test/resources/test_runner_services.xml-cd-legacy delete mode 100644 controller-server/src/test/resources/test_runner_services.xml-cd-osgi (limited to 'controller-server/src/test') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 11086ff7663..5cf554f2c01 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -521,22 +521,16 @@ public class InternalStepRunnerTest { } @Test - public void generates_correct_services_xml_using_osgi_based_runtime() { - generates_correct_services_xml("test_runner_services.xml-cd-osgi", true); + public void generates_correct_services_xml() { + generates_correct_services_xml("test_runner_services.xml-cd"); } - @Test - public void generates_correct_services_xml_using_legacy_runtime() { - generates_correct_services_xml("test_runner_services.xml-cd-legacy", false); - } - - private void generates_correct_services_xml(String filenameExpectedOutput, boolean useOsgiBasedRuntime) { + private void generates_correct_services_xml(String filenameExpectedOutput) { ControllerConfig.Steprunner.Testerapp config = new ControllerConfig.Steprunner.Testerapp.Builder().build(); assertFile(filenameExpectedOutput, new String(InternalStepRunner.servicesXml( true, false, - useOsgiBasedRuntime, new NodeResources(2, 12, 75, 1, NodeResources.DiskSpeed.fast, NodeResources.StorageType.local), config))); } diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd b/controller-server/src/test/resources/test_runner_services.xml-cd new file mode 100644 index 00000000000..634137e3fb6 --- /dev/null +++ b/controller-server/src/test/resources/test_runner_services.xml-cd @@ -0,0 +1,38 @@ + + + + + + + artifacts + 5120 + true + false + + + + + http://*/tester/v1/* + + + + + + + artifacts + true + + + + + + artifacts + true + + + + + + + + diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-legacy b/controller-server/src/test/resources/test_runner_services.xml-cd-legacy deleted file mode 100644 index c6046479934..00000000000 --- a/controller-server/src/test/resources/test_runner_services.xml-cd-legacy +++ /dev/null @@ -1,22 +0,0 @@ - - - - - - - artifacts - 5120 - true - false - - - - - http://*/tester/v1/* - - - - - - - diff --git a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi b/controller-server/src/test/resources/test_runner_services.xml-cd-osgi deleted file mode 100644 index 634137e3fb6..00000000000 --- a/controller-server/src/test/resources/test_runner_services.xml-cd-osgi +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - artifacts - 5120 - true - false - - - - - http://*/tester/v1/* - - - - - - - artifacts - true - - - - - - artifacts - true - - - - - - - - -- cgit v1.2.3 From 7c5d04b0eb540d1933713f34a53f6cb3d31e8517 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 19 Nov 2021 13:34:56 +0100 Subject: Only create ALIAS records when using exclusive routing --- .../vespa/hosted/controller/routing/RoutingPolicies.java | 13 ++++++++++--- .../com/yahoo/vespa/hosted/controller/ControllerTest.java | 8 ++++---- 2 files changed, 14 insertions(+), 7 deletions(-) (limited to 'controller-server/src/test') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 2a39ed08014..634d76c8449 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -215,7 +215,7 @@ public class RoutingPolicies { Application application = controller.applications().requireApplication(routingTable.keySet().iterator().next().application()); Map> targetWeights = targetWeights(application); - Map> targetsByEndpoint = new LinkedHashMap<>(); + Map> targetsByEndpoint = new LinkedHashMap<>(); for (Map.Entry> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); EndpointList endpoints = controller.routing().declaredEndpointsOf(application) @@ -236,13 +236,20 @@ public class RoutingPolicies { } WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), target.deployment().zoneId(), weight); - targetsByEndpoint.computeIfAbsent(endpoint.dnsName(), (k) -> new LinkedHashSet<>()) + targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()) .add(weightedAliasTarget); } } } targetsByEndpoint.forEach((applicationEndpoint, targets) -> { - controller.nameServiceForwarder().createAlias(RecordName.from(applicationEndpoint), targets, Priority.normal); + ZoneId targetZone = applicationEndpoint.targets().stream() + .map(Endpoint.Target::deployment) + .map(DeploymentId::zoneId) + .findFirst() + .get(); + nameServiceForwarderIn(targetZone).createAlias(RecordName.from(applicationEndpoint.dnsName()), + targets, + Priority.normal); }); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 9472801ef2c..58d034582cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -625,13 +625,13 @@ public class ControllerTest { .instances("beta,main") .region("us-west-1") .region("us-east-3") - .applicationEndpoint("a", "qrs", "us-west-1", + .applicationEndpoint("a", "default", "us-west-1", Map.of(InstanceName.from("beta"), 2, InstanceName.from("main"), 8)) - .applicationEndpoint("b", "qrs", "us-west-1", + .applicationEndpoint("b", "default", "us-west-1", Map.of(InstanceName.from("beta"), 1, InstanceName.from("main"), 1)) - .applicationEndpoint("c", "qrs", "us-east-3", + .applicationEndpoint("c", "default", "us-east-3", Map.of(InstanceName.from("beta"), 4, InstanceName.from("main"), 6)) .build(); @@ -644,7 +644,7 @@ public class ControllerTest { usEast, List.of("c--app1--tenant1.us-east-3-r.vespa.oath.cloud")); deploymentEndpoints.forEach((zone, endpointNames) -> { assertEquals("Endpoint names are passed to config server in " + zone, - Set.of(new ContainerEndpoint("qrs", "application", + Set.of(new ContainerEndpoint("default", "application", endpointNames)), tester.configServer().containerEndpoints().get(zone)); }); -- cgit v1.2.3 From 2747db0142641bf35d165d7d023450577e165775 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 19 Nov 2021 13:38:10 +0100 Subject: Disallow application endpoints with shared routing method --- .../vespa/hosted/controller/RoutingController.java | 25 +++++++++++++--------- .../hosted/controller/application/Endpoint.java | 7 +++++- .../vespa/hosted/controller/ControllerTest.java | 16 +++++++------- .../restapi/application/responses/deployment.json | 4 ++-- .../application/responses/prod-us-central-1.json | 4 ++-- 5 files changed, 33 insertions(+), 23 deletions(-) (limited to 'controller-server/src/test') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 3794b69c023..2f5b92ca4c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -145,16 +145,17 @@ public class RoutingController { .collect(Collectors.toMap(t -> new DeploymentId(application.id().instance(t.instance()), ZoneId.from(Environment.prod, t.region())), t -> t.weight())); - List availableRoutingMethods = routingMethodsOfAll(deployments.keySet(), deploymentSpec); - for (var routingMethod : availableRoutingMethods) { - endpoints.add(Endpoint.of(application.id()) - .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), - ClusterSpec.Id.from(declaredEndpoint.containerId()), - deployments) - .routingMethod(routingMethod) - .on(Port.fromRoutingMethod(routingMethod)) - .in(controller.system())); - } + // An application endpoint can only target a single zone, so we just pick the zone of any deployment target + ZoneId zone = deployments.keySet().iterator().next().zoneId(); + // Application endpoints are only supported when using direct routing methods + RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive; + endpoints.add(Endpoint.of(application.id()) + .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), + ClusterSpec.Id.from(declaredEndpoint.containerId()), + deployments) + .routingMethod(routingMethod) + .on(Port.fromRoutingMethod(routingMethod)) + .in(controller.system())); } return EndpointList.copyOf(endpoints); } @@ -354,6 +355,10 @@ public class RoutingController { Priority.normal)); } + private boolean usesSharedRouting(ZoneId zone) { + return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); + } + /** Returns the routing methods that are available across all given deployments */ private List routingMethodsOfAll(Collection deployments, DeploymentSpec deploymentSpec) { var deploymentsByMethod = new HashMap>(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index c736863a57e..aee7c1052be 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -60,7 +60,7 @@ public class Endpoint { this.instance = requireInstance(instanceName, scope); this.url = url; this.targets = List.copyOf(requireTargets(targets, application, instanceName, scope, certificateName)); - this.scope = scope; + this.scope = requireScope(scope, routingMethod); this.legacy = legacy; this.routingMethod = routingMethod; this.tls = port.tls; @@ -329,6 +329,11 @@ public class Endpoint { return instanceName; } + private static Scope requireScope(Scope scope, RoutingMethod routingMethod) { + if (scope == Scope.application && !routingMethod.isDirect()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not support " + scope + "-scoped endpoints"); + return scope; + } + private static List requireTargets(List targets, TenantAndApplicationId application, Optional instanceName, Scope scope, boolean certificateName) { if (!certificateName && targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); if (scope == Scope.zone && targets.size() != 1) throw new IllegalArgumentException("Exactly one target must be given for " + scope + " endpoints"); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 58d034582cd..be180f27af6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -640,8 +640,8 @@ public class ControllerTest { // Endpoint names are passed to each deployment DeploymentId usWest = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); DeploymentId usEast = context.deploymentIdIn(ZoneId.from("prod", "us-east-3")); - Map> deploymentEndpoints = Map.of(usWest, List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", "b--app1--tenant1.us-west-1-r.vespa.oath.cloud"), - usEast, List.of("c--app1--tenant1.us-east-3-r.vespa.oath.cloud")); + Map> deploymentEndpoints = Map.of(usWest, List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", "b.app1.tenant1.us-west-1-r.vespa.oath.cloud"), + usEast, List.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud")); deploymentEndpoints.forEach((zone, endpointNames) -> { assertEquals("Endpoint names are passed to config server in " + zone, Set.of(new ContainerEndpoint("default", "application", @@ -653,21 +653,21 @@ public class ControllerTest { // DNS records are created for each endpoint Set records = tester.controllerTester().nameService().records(); assertEquals(Set.of(new Record(Record.Type.CNAME, - RecordName.from("a--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + RecordName.from("a.app1.tenant1.us-west-1-r.vespa.oath.cloud"), RecordData.from("vip.prod.us-west-1.")), new Record(Record.Type.CNAME, - RecordName.from("b--app1--tenant1.us-west-1-r.vespa.oath.cloud"), + RecordName.from("b.app1.tenant1.us-west-1-r.vespa.oath.cloud"), RecordData.from("vip.prod.us-west-1.")), new Record(Record.Type.CNAME, - RecordName.from("c--app1--tenant1.us-east-3-r.vespa.oath.cloud"), + RecordName.from("c.app1.tenant1.us-east-3-r.vespa.oath.cloud"), RecordData.from("vip.prod.us-east-3."))), records); List endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application()) .scope(Endpoint.Scope.application) .mapToList(Endpoint::dnsName); - assertEquals(List.of("a--app1--tenant1.us-west-1-r.vespa.oath.cloud", - "b--app1--tenant1.us-west-1-r.vespa.oath.cloud", - "c--app1--tenant1.us-east-3-r.vespa.oath.cloud"), + assertEquals(List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", + "b.app1.tenant1.us-west-1-r.vespa.oath.cloud", + "c.app1.tenant1.us-east-3-r.vespa.oath.cloud"), endpointDnsNames); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json index fb6088f54b8..ab2a3bf945c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment.json @@ -24,9 +24,9 @@ { "cluster": "foo", "tls": true, - "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/", + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", "scope": "application", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false } ], diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json index 409e97b063c..62ad3a2db7e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/prod-us-central-1.json @@ -27,9 +27,9 @@ { "cluster": "foo", "tls": true, - "url": "https://a0--application1--tenant1.us-central-1-r.vespa.oath.cloud:4443/", + "url": "https://a0.application1.tenant1.us-central-1-r.vespa.oath.cloud/", "scope": "application", - "routingMethod": "shared", + "routingMethod": "sharedLayer4", "legacy": false } ], -- cgit v1.2.3 From ada377ef8e8cf29102a2ddb696332dca37f71d57 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 18 Nov 2021 15:52:01 +0100 Subject: Move rotation to routing package --- .../yahoo/vespa/hosted/controller/Instance.java | 2 +- .../vespa/hosted/controller/RoutingController.java | 4 +- .../controller/application/AssignedRotation.java | 2 +- .../controller/maintenance/MetricsReporter.java | 2 +- .../persistence/ApplicationSerializer.java | 6 +- .../restapi/application/ApplicationApiHandler.java | 6 +- .../vespa/hosted/controller/rotation/Rotation.java | 50 ----- .../hosted/controller/rotation/RotationId.java | 42 ---- .../hosted/controller/rotation/RotationLock.java | 25 --- .../controller/rotation/RotationRepository.java | 191 ------------------ .../hosted/controller/rotation/RotationState.java | 20 -- .../hosted/controller/rotation/RotationStatus.java | 103 ---------- .../controller/routing/rotation/Rotation.java | 50 +++++ .../controller/routing/rotation/RotationId.java | 42 ++++ .../controller/routing/rotation/RotationLock.java | 25 +++ .../routing/rotation/RotationRepository.java | 191 ++++++++++++++++++ .../controller/routing/rotation/RotationState.java | 20 ++ .../routing/rotation/RotationStatus.java | 103 ++++++++++ .../vespa/hosted/controller/ControllerTest.java | 4 +- .../persistence/ApplicationSerializerTest.java | 6 +- .../rotation/RotationRepositoryTest.java | 218 --------------------- .../routing/rotation/RotationRepositoryTest.java | 218 +++++++++++++++++++++ 22 files changed, 665 insertions(+), 665 deletions(-) delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java delete mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java delete mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java create mode 100644 controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java (limited to 'controller-server/src/test') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java index ea2bcfcac4b..6e31c93dbdd 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Instance.java @@ -14,7 +14,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentActivity; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import java.time.Instant; import java.util.Collection; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 2f5b92ca4c1..c832b6672d0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -31,8 +31,8 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java index 1596456b7cc..ab9304e75f3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/AssignedRotation.java @@ -3,7 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.RegionName; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import java.util.Collection; import java.util.Objects; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java index 47df7a9da92..2939d10f99e 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/MetricsReporter.java @@ -16,7 +16,7 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import com.yahoo.vespa.hosted.controller.deployment.JobList; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.versions.NodeVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java index e8a7f7729fb..4b060846090 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializer.java @@ -31,9 +31,9 @@ import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import java.security.PublicKey; import java.time.Instant; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5cd5a70e4a4..5d4b45fa82b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -97,9 +97,9 @@ import com.yahoo.vespa.hosted.controller.maintenance.ResourceMeterMaintainer; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.SupportAccessSerializer; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java deleted file mode 100644 index ca5d2d5915f..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/Rotation.java +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.text.Text; -import java.util.Objects; - -/** - * Represents a global routing rotation. - * - * @author mpolden - */ -public class Rotation { - - private final RotationId id; - private final String name; - - public Rotation(RotationId id, String name) { - this.id = Objects.requireNonNull(id); - this.name = Objects.requireNonNull(name); - } - - /** The ID of the allocated rotation. This value is generated by global routing system */ - public RotationId id() { - return id; - } - - /** The global rotation FQDN */ - public String name() { - return name; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof Rotation)) return false; - final Rotation rotation = (Rotation) o; - return id().equals(rotation.id()) && name().equals(rotation.name()); - } - - @Override - public int hashCode() { - return Objects.hash(id(), name()); - } - - @Override - public String toString() { - return Text.format("rotation %s -> %s", id().asString(), name()); - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java deleted file mode 100644 index 2b75777fbbd..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationId.java +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import java.util.Objects; - -/** - * ID of a global rotation. - * - * @author mpolden - */ -public class RotationId { - - private final String id; - - public RotationId(String id) { - this.id = id; - } - - /** Rotation ID, e.g. rotation-42.vespa.global.routing */ - public String asString() { - return id; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RotationId that = (RotationId) o; - return Objects.equals(id, that.id); - } - - @Override - public int hashCode() { - return Objects.hash(id); - } - - @Override - public String toString() { - return "rotation ID " + id; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java deleted file mode 100644 index fe9280b1193..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationLock.java +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.vespa.curator.Lock; - -import java.util.Objects; - -/** - * A lock for the rotation repository. This is a type-safe wrapper for a curator lock. - * - * @author mpolden - */ -public class RotationLock implements AutoCloseable { - - private final Lock lock; - - RotationLock(Lock lock) { - this.lock = Objects.requireNonNull(lock, "lock cannot be null"); - } - - @Override - public void close() { - lock.close(); - } -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java deleted file mode 100644 index 5b24f39717b..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java +++ /dev/null @@ -1,191 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.config.application.api.DeploymentInstanceSpec; -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.application.api.Endpoint; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; -import com.yahoo.text.Text; -import com.yahoo.vespa.hosted.controller.ApplicationController; -import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.Set; -import java.util.function.Function; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static java.util.stream.Collectors.collectingAndThen; - -/** - * The rotation repository offers global rotations to Vespa applications. - * - * The list of rotations comes from RotationsConfig, which is set in the controller's services.xml. - * - * @author Oyvind Gronnesby - * @author mpolden - */ -public class RotationRepository { - - private static final Logger log = Logger.getLogger(RotationRepository.class.getName()); - - private final Map allRotations; - private final ApplicationController applications; - private final CuratorDb curator; - - public RotationRepository(RotationsConfig rotationsConfig, ApplicationController applications, CuratorDb curator) { - this.allRotations = from(rotationsConfig); - this.applications = applications; - this.curator = curator; - } - - /** Acquire a exclusive lock for this */ - public RotationLock lock() { - return new RotationLock(curator.lockRotations()); - } - - /** Get rotation by given rotationId */ - public Optional getRotation(RotationId rotationId) { - return Optional.of(allRotations.get(rotationId)); - } - - /** - * Returns a single rotation for the given application. This is only used when a rotation is assigned through the - * use of a global service ID. - * - * If a rotation is already assigned to the application, that rotation will be returned. - * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. - * - * @param instanceSpec the instance deployment spec - * @param instance the instance requesting a rotation - * @param lock lock which must be acquired by the caller - */ - private AssignedRotation assignRotationTo(String globalServiceId, DeploymentInstanceSpec instanceSpec, - Instance instance, RotationLock lock) { - RotationId rotation; - if (instance.rotations().isEmpty()) { - rotation = findAvailableRotation(instance.id(), lock).id(); - } else { - rotation = instance.rotations().get(0).rotationId(); - } - var productionRegions = instanceSpec.zones().stream() - .filter(zone -> zone.environment().isProduction()) - .flatMap(zone -> zone.region().stream()) - .collect(Collectors.toSet()); - if (productionRegions.size() < 2) { - throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + - "in instance '" + instance.name() + "'"); - } - return new AssignedRotation(new ClusterSpec.Id(globalServiceId), - EndpointId.defaultId(), - rotation, - productionRegions); - } - - /** - * Returns rotation assignments for all endpoints in application. - * - * If rotations are already assigned, these will be returned. - * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. - * This method supports both global-service-id as well as the new endpoints tag. - * - * @param deploymentSpec The deployment spec of the application - * @param instance The application requesting rotations - * @param lock Lock which by acquired by the caller - * @return List of rotation assignments - either new or existing - */ - public List getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { - // Skip assignment if no rotations are configured in this system - if (allRotations.isEmpty()) { - return List.of(); - } - - // Only allow one kind of configuration syntax - var instanceSpec = deploymentSpec.requireInstance(instance.name()); - if ( instanceSpec.globalServiceId().isPresent() - && ! instanceSpec.endpoints().isEmpty()) { - throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); - } - - // Support legacy global-service-id - if (instanceSpec.globalServiceId().isPresent()) { - return List.of(assignRotationTo(instanceSpec.globalServiceId().get(), instanceSpec, instance, lock)); - } - - return assignRotationsTo(instanceSpec.endpoints(), instance, lock); - } - - private List assignRotationsTo(List endpoints, Instance instance, RotationLock lock) { - if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign. - var availableRotations = new ArrayList<>(availableRotations(lock).values()); - var assignedRotationsByEndpointId = instance.rotations().stream() - .collect(Collectors.toMap(AssignedRotation::endpointId, - Function.identity())); - var assignments = new ArrayList(); - for (var endpoint : endpoints) { - var endpointId = EndpointId.of(endpoint.endpointId()); - var assignedRotation = assignedRotationsByEndpointId.get(endpointId); - RotationId rotationId; - if (assignedRotation == null) { // No rotation is assigned to this endpoint - rotationId = requireNonEmpty(availableRotations).remove(0).id(); - } else { // Rotation already assigned to this endpoint, reuse it - rotationId = assignedRotation.rotationId(); - } - assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, Set.copyOf(endpoint.regions()))); - } - return Collections.unmodifiableList(assignments); - } - - /** - * Returns all unassigned rotations - * @param lock Lock which must be acquired by the caller - */ - public Map availableRotations(@SuppressWarnings("unused") RotationLock lock) { - List assignedRotations = applications.asList().stream() - .flatMap(application -> application.instances().values().stream()) - .flatMap(instance -> instance.rotations().stream()) - .map(AssignedRotation::rotationId) - .collect(Collectors.toList()); - Map unassignedRotations = new LinkedHashMap<>(this.allRotations); - assignedRotations.forEach(unassignedRotations::remove); - return Collections.unmodifiableMap(unassignedRotations); - } - - private Rotation findAvailableRotation(ApplicationId id, RotationLock lock) { - Map availableRotations = availableRotations(lock); - // Return first available rotation - RotationId rotation = requireNonEmpty(availableRotations.keySet()).iterator().next(); - log.info(Text.format("Offering %s to application %s", rotation, id)); - return allRotations.get(rotation); - } - - /** Returns a immutable map of rotation ID to rotation sorted by rotation ID */ - private static Map from(RotationsConfig rotationConfig) { - return rotationConfig.rotations().entrySet().stream() - .map(entry -> new Rotation(new RotationId(entry.getKey()), entry.getValue().trim())) - .sorted(Comparator.comparing(rotation -> rotation.id().asString())) - .collect(collectingAndThen(Collectors.toMap(Rotation::id, - rotation -> rotation, - (k, v) -> v, - LinkedHashMap::new), - Collections::unmodifiableMap)); - } - - private static > T requireNonEmpty(T rotations) { - if (rotations.isEmpty()) throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation"); - return rotations; - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java deleted file mode 100644 index 032f01433b3..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationState.java +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -/** - * The possible states of a global rotation. - * - * @author mpolden - */ -public enum RotationState { - - /** Rotation has status 'in' and is receiving traffic */ - in, - - /** Rotation has status 'out' and is *NOT* receiving traffic */ - out, - - /** Rotation status is currently unknown, or no global rotation has been assigned */ - unknown - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java deleted file mode 100644 index 1ddbd640e53..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationStatus.java +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.hosted.controller.application.Deployment; - -import java.time.Instant; -import java.util.Map; -import java.util.Objects; - -/** - * The status of all rotations assigned to an application. - * - * @author mpolden - */ -public class RotationStatus { - - public static final RotationStatus EMPTY = new RotationStatus(Map.of()); - - private final Map status; - - private RotationStatus(Map status) { - this.status = Map.copyOf(Objects.requireNonNull(status)); - } - - public Map asMap() { - return status; - } - - /** Get targets of given rotation, if any */ - public Targets of(RotationId rotation) { - return status.getOrDefault(rotation, Targets.NONE); - } - - /** Get status of deployment in given rotation, if any */ - public RotationState of(RotationId rotation, Deployment deployment) { - return of(rotation).asMap().entrySet().stream() - .filter(kv -> kv.getKey().equals(deployment.zone())) - .map(Map.Entry::getValue) - .findFirst() - .orElse(RotationState.unknown); - } - - @Override - public String toString() { - return "rotation status " + status; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - RotationStatus that = (RotationStatus) o; - return status.equals(that.status); - } - - @Override - public int hashCode() { - return Objects.hash(status); - } - - public static RotationStatus from(Map targets) { - return targets.isEmpty() ? EMPTY : new RotationStatus(targets); - } - - /** Targets of a rotation */ - public static class Targets { - - public static final Targets NONE = new Targets(Map.of(), Instant.EPOCH); - - private final Map targets; - private final Instant lastUpdated; - - public Targets(Map targets, Instant lastUpdated) { - this.targets = Map.copyOf(Objects.requireNonNull(targets, "states must be non-null")); - this.lastUpdated = Objects.requireNonNull(lastUpdated, "lastUpdated must be non-null"); - } - - public Map asMap() { - return targets; - } - - public Instant lastUpdated() { - return lastUpdated; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - Targets targets1 = (Targets) o; - return targets.equals(targets1.targets) && - lastUpdated.equals(targets1.lastUpdated); - } - - @Override - public int hashCode() { - return Objects.hash(targets, lastUpdated); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java new file mode 100644 index 00000000000..0cf7101cac0 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/Rotation.java @@ -0,0 +1,50 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.text.Text; +import java.util.Objects; + +/** + * Represents a global routing rotation. + * + * @author mpolden + */ +public class Rotation { + + private final RotationId id; + private final String name; + + public Rotation(RotationId id, String name) { + this.id = Objects.requireNonNull(id); + this.name = Objects.requireNonNull(name); + } + + /** The ID of the allocated rotation. This value is generated by global routing system */ + public RotationId id() { + return id; + } + + /** The global rotation FQDN */ + public String name() { + return name; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof Rotation)) return false; + final Rotation rotation = (Rotation) o; + return id().equals(rotation.id()) && name().equals(rotation.name()); + } + + @Override + public int hashCode() { + return Objects.hash(id(), name()); + } + + @Override + public String toString() { + return Text.format("rotation %s -> %s", id().asString(), name()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java new file mode 100644 index 00000000000..4d97962a40a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationId.java @@ -0,0 +1,42 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import java.util.Objects; + +/** + * ID of a global rotation. + * + * @author mpolden + */ +public class RotationId { + + private final String id; + + public RotationId(String id) { + this.id = id; + } + + /** Rotation ID, e.g. rotation-42.vespa.global.routing */ + public String asString() { + return id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RotationId that = (RotationId) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + public String toString() { + return "rotation ID " + id; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java new file mode 100644 index 00000000000..36a43f80e9a --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationLock.java @@ -0,0 +1,25 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.vespa.curator.Lock; + +import java.util.Objects; + +/** + * A lock for the rotation repository. This is a type-safe wrapper for a curator lock. + * + * @author mpolden + */ +public class RotationLock implements AutoCloseable { + + private final Lock lock; + + RotationLock(Lock lock) { + this.lock = Objects.requireNonNull(lock, "lock cannot be null"); + } + + @Override + public void close() { + lock.close(); + } +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java new file mode 100644 index 00000000000..961fdc6dd9c --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepository.java @@ -0,0 +1,191 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.config.application.api.DeploymentInstanceSpec; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.Endpoint; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.text.Text; +import com.yahoo.vespa.hosted.controller.ApplicationController; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static java.util.stream.Collectors.collectingAndThen; + +/** + * The rotation repository offers global rotations to Vespa applications. + * + * The list of rotations comes from RotationsConfig, which is set in the controller's services.xml. + * + * @author Oyvind Gronnesby + * @author mpolden + */ +public class RotationRepository { + + private static final Logger log = Logger.getLogger(RotationRepository.class.getName()); + + private final Map allRotations; + private final ApplicationController applications; + private final CuratorDb curator; + + public RotationRepository(RotationsConfig rotationsConfig, ApplicationController applications, CuratorDb curator) { + this.allRotations = from(rotationsConfig); + this.applications = applications; + this.curator = curator; + } + + /** Acquire a exclusive lock for this */ + public RotationLock lock() { + return new RotationLock(curator.lockRotations()); + } + + /** Get rotation by given rotationId */ + public Optional getRotation(RotationId rotationId) { + return Optional.of(allRotations.get(rotationId)); + } + + /** + * Returns a single rotation for the given application. This is only used when a rotation is assigned through the + * use of a global service ID. + * + * If a rotation is already assigned to the application, that rotation will be returned. + * If no rotation is assigned, return an available rotation. The caller is responsible for assigning the rotation. + * + * @param instanceSpec the instance deployment spec + * @param instance the instance requesting a rotation + * @param lock lock which must be acquired by the caller + */ + private AssignedRotation assignRotationTo(String globalServiceId, DeploymentInstanceSpec instanceSpec, + Instance instance, RotationLock lock) { + RotationId rotation; + if (instance.rotations().isEmpty()) { + rotation = findAvailableRotation(instance.id(), lock).id(); + } else { + rotation = instance.rotations().get(0).rotationId(); + } + var productionRegions = instanceSpec.zones().stream() + .filter(zone -> zone.environment().isProduction()) + .flatMap(zone -> zone.region().stream()) + .collect(Collectors.toSet()); + if (productionRegions.size() < 2) { + throw new IllegalArgumentException("global-service-id is set but less than 2 prod zones are defined " + + "in instance '" + instance.name() + "'"); + } + return new AssignedRotation(new ClusterSpec.Id(globalServiceId), + EndpointId.defaultId(), + rotation, + productionRegions); + } + + /** + * Returns rotation assignments for all endpoints in application. + * + * If rotations are already assigned, these will be returned. + * If rotations are not assigned, a new assignment will be created taking new rotations from the repository. + * This method supports both global-service-id as well as the new endpoints tag. + * + * @param deploymentSpec The deployment spec of the application + * @param instance The application requesting rotations + * @param lock Lock which by acquired by the caller + * @return List of rotation assignments - either new or existing + */ + public List getOrAssignRotations(DeploymentSpec deploymentSpec, Instance instance, RotationLock lock) { + // Skip assignment if no rotations are configured in this system + if (allRotations.isEmpty()) { + return List.of(); + } + + // Only allow one kind of configuration syntax + var instanceSpec = deploymentSpec.requireInstance(instance.name()); + if ( instanceSpec.globalServiceId().isPresent() + && ! instanceSpec.endpoints().isEmpty()) { + throw new IllegalArgumentException("Cannot provision rotations with both global-service-id and 'endpoints'"); + } + + // Support legacy global-service-id + if (instanceSpec.globalServiceId().isPresent()) { + return List.of(assignRotationTo(instanceSpec.globalServiceId().get(), instanceSpec, instance, lock)); + } + + return assignRotationsTo(instanceSpec.endpoints(), instance, lock); + } + + private List assignRotationsTo(List endpoints, Instance instance, RotationLock lock) { + if (endpoints.isEmpty()) return List.of(); // No endpoints declared, nothing to assign. + var availableRotations = new ArrayList<>(availableRotations(lock).values()); + var assignedRotationsByEndpointId = instance.rotations().stream() + .collect(Collectors.toMap(AssignedRotation::endpointId, + Function.identity())); + var assignments = new ArrayList(); + for (var endpoint : endpoints) { + var endpointId = EndpointId.of(endpoint.endpointId()); + var assignedRotation = assignedRotationsByEndpointId.get(endpointId); + RotationId rotationId; + if (assignedRotation == null) { // No rotation is assigned to this endpoint + rotationId = requireNonEmpty(availableRotations).remove(0).id(); + } else { // Rotation already assigned to this endpoint, reuse it + rotationId = assignedRotation.rotationId(); + } + assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, Set.copyOf(endpoint.regions()))); + } + return Collections.unmodifiableList(assignments); + } + + /** + * Returns all unassigned rotations + * @param lock Lock which must be acquired by the caller + */ + public Map availableRotations(@SuppressWarnings("unused") RotationLock lock) { + List assignedRotations = applications.asList().stream() + .flatMap(application -> application.instances().values().stream()) + .flatMap(instance -> instance.rotations().stream()) + .map(AssignedRotation::rotationId) + .collect(Collectors.toList()); + Map unassignedRotations = new LinkedHashMap<>(this.allRotations); + assignedRotations.forEach(unassignedRotations::remove); + return Collections.unmodifiableMap(unassignedRotations); + } + + private Rotation findAvailableRotation(ApplicationId id, RotationLock lock) { + Map availableRotations = availableRotations(lock); + // Return first available rotation + RotationId rotation = requireNonEmpty(availableRotations.keySet()).iterator().next(); + log.info(Text.format("Offering %s to application %s", rotation, id)); + return allRotations.get(rotation); + } + + /** Returns a immutable map of rotation ID to rotation sorted by rotation ID */ + private static Map from(RotationsConfig rotationConfig) { + return rotationConfig.rotations().entrySet().stream() + .map(entry -> new Rotation(new RotationId(entry.getKey()), entry.getValue().trim())) + .sorted(Comparator.comparing(rotation -> rotation.id().asString())) + .collect(collectingAndThen(Collectors.toMap(Rotation::id, + rotation -> rotation, + (k, v) -> v, + LinkedHashMap::new), + Collections::unmodifiableMap)); + } + + private static > T requireNonEmpty(T rotations) { + if (rotations.isEmpty()) throw new IllegalStateException("Hosted Vespa ran out of rotations, unable to assign rotation"); + return rotations; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java new file mode 100644 index 00000000000..19e816a0b51 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationState.java @@ -0,0 +1,20 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +/** + * The possible states of a global rotation. + * + * @author mpolden + */ +public enum RotationState { + + /** Rotation has status 'in' and is receiving traffic */ + in, + + /** Rotation has status 'out' and is *NOT* receiving traffic */ + out, + + /** Rotation status is currently unknown, or no global rotation has been assigned */ + unknown + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java new file mode 100644 index 00000000000..6d95ad9a230 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationStatus.java @@ -0,0 +1,103 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Deployment; + +import java.time.Instant; +import java.util.Map; +import java.util.Objects; + +/** + * The status of all rotations assigned to an application. + * + * @author mpolden + */ +public class RotationStatus { + + public static final RotationStatus EMPTY = new RotationStatus(Map.of()); + + private final Map status; + + private RotationStatus(Map status) { + this.status = Map.copyOf(Objects.requireNonNull(status)); + } + + public Map asMap() { + return status; + } + + /** Get targets of given rotation, if any */ + public Targets of(RotationId rotation) { + return status.getOrDefault(rotation, Targets.NONE); + } + + /** Get status of deployment in given rotation, if any */ + public RotationState of(RotationId rotation, Deployment deployment) { + return of(rotation).asMap().entrySet().stream() + .filter(kv -> kv.getKey().equals(deployment.zone())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(RotationState.unknown); + } + + @Override + public String toString() { + return "rotation status " + status; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RotationStatus that = (RotationStatus) o; + return status.equals(that.status); + } + + @Override + public int hashCode() { + return Objects.hash(status); + } + + public static RotationStatus from(Map targets) { + return targets.isEmpty() ? EMPTY : new RotationStatus(targets); + } + + /** Targets of a rotation */ + public static class Targets { + + public static final Targets NONE = new Targets(Map.of(), Instant.EPOCH); + + private final Map targets; + private final Instant lastUpdated; + + public Targets(Map targets, Instant lastUpdated) { + this.targets = Map.copyOf(Objects.requireNonNull(targets, "states must be non-null")); + this.lastUpdated = Objects.requireNonNull(lastUpdated, "lastUpdated must be non-null"); + } + + public Map asMap() { + return targets; + } + + public Instant lastUpdated() { + return lastUpdated; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Targets targets1 = (Targets) o; + return targets.equals(targets1.targets) && + lastUpdated.equals(targets1.lastUpdated); + } + + @Override + public int hashCode() { + return Objects.hash(targets, lastUpdated); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index be180f27af6..132e6caa3ca 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -38,8 +38,8 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java index f1421b5affd..b33f8f6f7e7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ApplicationSerializerTest.java @@ -23,9 +23,9 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.metric.ApplicationMetrics; -import com.yahoo.vespa.hosted.controller.rotation.RotationId; -import com.yahoo.vespa.hosted.controller.rotation.RotationState; -import com.yahoo.vespa.hosted.controller.rotation.RotationStatus; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import org.junit.Test; import java.nio.file.Files; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java deleted file mode 100644 index e7c2eacbd02..00000000000 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.rotation; - -import com.yahoo.config.provision.RegionName; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; -import com.yahoo.vespa.hosted.controller.application.AssignedRotation; -import com.yahoo.vespa.hosted.controller.application.SystemApplication; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; -import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; -import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; -import org.junit.Test; - -import java.net.URI; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -/** - * @author Oyvind Gronnesby - * @author mpolden - */ -public class RotationRepositoryTest { - - private static final RotationsConfig rotationsConfig = new RotationsConfig( - new RotationsConfig.Builder() - .rotations("foo-1", "foo-1.com") - .rotations("foo-2", "foo-2.com") - ); - - private static final RotationsConfig rotationsConfigWhitespaces = new RotationsConfig( - new RotationsConfig.Builder() - .rotations("foo-1", "\n \t foo-1.com \n") - .rotations("foo-2", "foo-2.com") - ); - - private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("us-east-3") - .region("us-west-1") - .build(); - - private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig, SystemName.main)); - private final RotationRepository repository = tester.controller().routing().rotations(); - private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default"); - - @Test - public void assigns_and_reuses_rotation() { - // Deploying assigns a rotation - application.submit(applicationPackage).deploy(); - Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); - - assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); - assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); - try (RotationLock lock = repository.lock()) { - List rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), - application.instance(), - lock); - assertSingleRotation(expected, rotations, repository); - assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")), - application.instance().rotations().get(0).regions()); - } - - // Submitting once more assigns same rotation - application.submit(applicationPackage).deploy(); - assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); - - // Adding region updates rotation - var applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("us-east-3") - .region("us-west-1") - .region("us-central-1") - .build(); - application.submit(applicationPackage).deploy(); - assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"), - RegionName.from("us-central-1")), - application.instance().rotations().get(0).regions()); - } - - @Test - public void strips_whitespace_in_rotation_fqdn() { - var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces, SystemName.main)); - RotationRepository repository = tester.controller().routing().rotations(); - var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); - - application2.submit(applicationPackage); - - try (RotationLock lock = repository.lock()) { - List rotations = repository.getOrAssignRotations(application2.application().deploymentSpec(), application2.instance(), lock); - Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); - assertSingleRotation(assignedRotation, rotations, repository); - } - } - - @Test - public void out_of_rotations() { - // Assigns 1 rotation - application.submit(applicationPackage).deploy(); - - // Assigns 1 more - var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); - application2.submit(applicationPackage).deploy(); - - // We're now out of rotations and next deployment fails - var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); - application3.submit(applicationPackage) - .runJobExpectingFailure(JobType.systemTest, Optional.of("out of rotations")); - } - - @Test - public void too_few_zones() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("us-east-3") - .build(); - application.submit(applicationPackage).runJobExpectingFailure(JobType.systemTest, Optional.of("less than 2 prod zones are defined")); - } - - @Test - public void no_rotation_assigned_for_application_without_service_id() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .region("us-east-3") - .region("us-west-1") - .build(); - application.submit(applicationPackage); - assertTrue(application.instance().rotations().isEmpty()); - } - - @Test - public void prefixes_system_when_not_main() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .globalServiceId("foo") - .region("cd-us-east-1") - .region("cd-us-west-1") - .build(); - var zones = List.of( - ZoneApiMock.fromId("test.cd-us-west-1"), - ZoneApiMock.fromId("staging.cd-us-west-1"), - ZoneApiMock.fromId("prod.cd-us-east-1"), - ZoneApiMock.fromId("prod.cd-us-west-1")); - tester.controllerTester().zoneRegistry() - .setZones(zones) - .setRoutingMethod(zones, RoutingMethod.shared) - .setSystemName(SystemName.cd); - tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); - var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); - application2.submit(applicationPackage).deploy(); - assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); - assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", - tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); - } - - @Test - public void multiple_instances_with_similar_global_service_id() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .instances("instance1,instance2") - .region("us-central-1") - .parallel("us-west-1", "us-east-3") - .globalServiceId("global") - .build(); - var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") - .submit(applicationPackage) - .deploy(); - var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); - assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); - assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); - } - - @Test - public void multiple_instances_with_similar_endpoints() { - ApplicationPackage applicationPackage = new ApplicationPackageBuilder() - .instances("instance1,instance2") - .region("us-central-1") - .parallel("us-west-1", "us-east-3") - .endpoint("default", "foo", "us-central-1", "us-west-1") - .build(); - var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") - .submit(applicationPackage) - .deploy(); - var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); - - assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); - assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); - - assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); - assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), - tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); - } - - private void assertSingleRotation(Rotation expected, List assignedRotations, RotationRepository repository) { - assertEquals(1, assignedRotations.size()); - var rotationId = assignedRotations.get(0).rotationId(); - var rotation = repository.getRotation(rotationId); - assertTrue(rotationId + " exists", rotation.isPresent()); - assertEquals(expected, rotation.get()); - } - - private static List rotationIds(List assignedRotations) { - return assignedRotations.stream().map(AssignedRotation::rotationId).collect(Collectors.toUnmodifiableList()); - } - -} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java new file mode 100644 index 00000000000..9a3ac8b547d --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/rotation/RotationRepositoryTest.java @@ -0,0 +1,218 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.rotation; + +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.junit.Test; + +import java.net.URI; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author Oyvind Gronnesby + * @author mpolden + */ +public class RotationRepositoryTest { + + private static final RotationsConfig rotationsConfig = new RotationsConfig( + new RotationsConfig.Builder() + .rotations("foo-1", "foo-1.com") + .rotations("foo-2", "foo-2.com") + ); + + private static final RotationsConfig rotationsConfigWhitespaces = new RotationsConfig( + new RotationsConfig.Builder() + .rotations("foo-1", "\n \t foo-1.com \n") + .rotations("foo-2", "foo-2.com") + ); + + private static final ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .region("us-west-1") + .build(); + + private final DeploymentTester tester = new DeploymentTester(new ControllerTester(rotationsConfig, SystemName.main)); + private final RotationRepository repository = tester.controller().routing().rotations(); + private final DeploymentContext application = tester.newDeploymentContext("tenant1", "app1", "default"); + + @Test + public void assigns_and_reuses_rotation() { + // Deploying assigns a rotation + application.submit(applicationPackage).deploy(); + Rotation expected = new Rotation(new RotationId("foo-1"), "foo-1.com"); + + assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); + assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(application.instanceId()).primary().get().url()); + try (RotationLock lock = repository.lock()) { + List rotations = repository.getOrAssignRotations(application.application().deploymentSpec(), + application.instance(), + lock); + assertSingleRotation(expected, rotations, repository); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3")), + application.instance().rotations().get(0).regions()); + } + + // Submitting once more assigns same rotation + application.submit(applicationPackage).deploy(); + assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations())); + + // Adding region updates rotation + var applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .region("us-west-1") + .region("us-central-1") + .build(); + application.submit(applicationPackage).deploy(); + assertEquals(Set.of(RegionName.from("us-west-1"), RegionName.from("us-east-3"), + RegionName.from("us-central-1")), + application.instance().rotations().get(0).regions()); + } + + @Test + public void strips_whitespace_in_rotation_fqdn() { + var tester = new DeploymentTester(new ControllerTester(rotationsConfigWhitespaces, SystemName.main)); + RotationRepository repository = tester.controller().routing().rotations(); + var application2 = tester.newDeploymentContext("tenant1", "app2", "default"); + + application2.submit(applicationPackage); + + try (RotationLock lock = repository.lock()) { + List rotations = repository.getOrAssignRotations(application2.application().deploymentSpec(), application2.instance(), lock); + Rotation assignedRotation = new Rotation(new RotationId("foo-1"), "foo-1.com"); + assertSingleRotation(assignedRotation, rotations, repository); + } + } + + @Test + public void out_of_rotations() { + // Assigns 1 rotation + application.submit(applicationPackage).deploy(); + + // Assigns 1 more + var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); + application2.submit(applicationPackage).deploy(); + + // We're now out of rotations and next deployment fails + var application3 = tester.newDeploymentContext("tenant3", "app3", "default"); + application3.submit(applicationPackage) + .runJobExpectingFailure(JobType.systemTest, Optional.of("out of rotations")); + } + + @Test + public void too_few_zones() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("us-east-3") + .build(); + application.submit(applicationPackage).runJobExpectingFailure(JobType.systemTest, Optional.of("less than 2 prod zones are defined")); + } + + @Test + public void no_rotation_assigned_for_application_without_service_id() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .region("us-east-3") + .region("us-west-1") + .build(); + application.submit(applicationPackage); + assertTrue(application.instance().rotations().isEmpty()); + } + + @Test + public void prefixes_system_when_not_main() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .globalServiceId("foo") + .region("cd-us-east-1") + .region("cd-us-west-1") + .build(); + var zones = List.of( + ZoneApiMock.fromId("test.cd-us-west-1"), + ZoneApiMock.fromId("staging.cd-us-west-1"), + ZoneApiMock.fromId("prod.cd-us-east-1"), + ZoneApiMock.fromId("prod.cd-us-west-1")); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.shared) + .setSystemName(SystemName.cd); + tester.configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), SystemApplication.notController()); + var application2 = tester.newDeploymentContext("tenant2", "app2", "default"); + application2.submit(applicationPackage).deploy(); + assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations())); + assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/", + tester.controller().routing().readDeclaredEndpointsOf(application2.instanceId()).primary().get().url().toString()); + } + + @Test + public void multiple_instances_with_similar_global_service_id() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1,instance2") + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .globalServiceId("global") + .build(); + var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") + .submit(applicationPackage) + .deploy(); + var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); + assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); + assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); + assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + } + + @Test + public void multiple_instances_with_similar_endpoints() { + ApplicationPackage applicationPackage = new ApplicationPackageBuilder() + .instances("instance1,instance2") + .region("us-central-1") + .parallel("us-west-1", "us-east-3") + .endpoint("default", "foo", "us-central-1", "us-west-1") + .build(); + var instance1 = tester.newDeploymentContext("tenant1", "application1", "instance1") + .submit(applicationPackage) + .deploy(); + var instance2 = tester.newDeploymentContext("tenant1", "application1", "instance2"); + + assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations())); + assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations())); + + assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance1.instanceId()).primary().get().url()); + assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"), + tester.controller().routing().readDeclaredEndpointsOf(instance2.instanceId()).primary().get().url()); + } + + private void assertSingleRotation(Rotation expected, List assignedRotations, RotationRepository repository) { + assertEquals(1, assignedRotations.size()); + var rotationId = assignedRotations.get(0).rotationId(); + var rotation = repository.getRotation(rotationId); + assertTrue(rotationId + " exists", rotation.isPresent()); + assertEquals(expected, rotation.get()); + } + + private static List rotationIds(List assignedRotations) { + return assignedRotations.stream().map(AssignedRotation::rotationId).collect(Collectors.toUnmodifiableList()); + } + +} -- cgit v1.2.3 From ad30cfaba7b1ba6acacd5e105dc8cbdea653a4ff Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 19 Nov 2021 09:32:33 +0100 Subject: Configure all routing variants through a RoutingContext --- .../api/application/v4/model/EndpointStatus.java | 3 +- .../hosted/controller/ApplicationController.java | 27 ++-- .../vespa/hosted/controller/RoutingController.java | 107 +++++++------- .../controller/deployment/InternalStepRunner.java | 17 +-- .../maintenance/SystemRoutingPolicyMaintainer.java | 4 +- .../restapi/application/ApplicationApiHandler.java | 53 +++---- .../restapi/routing/RoutingApiHandler.java | 117 ++++------------ .../routing/context/DeploymentRoutingContext.java | 154 +++++++++++++++++++++ .../routing/context/ExclusiveRoutingContext.java | 41 ++++++ .../controller/routing/context/RoutingContext.java | 23 +++ .../routing/context/SharedRoutingContext.java | 48 +++++++ .../vespa/hosted/controller/ControllerTest.java | 23 ++- .../restapi/application/ApplicationApiTest.java | 14 +- .../application/responses/global-rotation-get.json | 2 +- 14 files changed, 409 insertions(+), 224 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java (limited to 'controller-server/src/test') diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java index 55a7af45fd2..2f1b93158ab 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/EndpointStatus.java @@ -50,9 +50,10 @@ public class EndpointStatus { } /** - * @return The epoch for when this status became active + * @return The epoch for when this status became active, in seconds */ public long getEpoch() { return epoch; } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index df0c727ec24..9e7c614d4e8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -351,6 +351,7 @@ public class ApplicationController { TenantAndApplicationId applicationId = TenantAndApplicationId.from(job.application()); ZoneId zone = job.type().zone(controller.system()); + DeploymentId deployment = new DeploymentId(job.application(), zone); try (Lock deploymentLock = lockForDeployment(job.application(), zone)) { Set containerEndpoints; @@ -364,7 +365,7 @@ public class ApplicationController { Version platform = run.versions().sourcePlatform().filter(__ -> deploySourceVersions).orElse(run.versions().targetPlatform()); ApplicationVersion revision = run.versions().sourceApplication().filter(__ -> deploySourceVersions).orElse(run.versions().targetApplication()); - ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(new DeploymentId(job.application(), zone), revision)); + ApplicationPackage applicationPackage = new ApplicationPackage(applicationStore.get(deployment, revision)); try (Lock lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); @@ -376,8 +377,7 @@ public class ApplicationController { applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get()); endpointCertificateMetadata = endpointCertificates.getMetadata(instance, zone, applicationPackage.deploymentSpec()); - - containerEndpoints = controller.routing().containerEndpointsOf(application, job.application().instance(), zone); + containerEndpoints = controller.routing().of(deployment).prepare(application); } // Release application lock while doing the deployment, which is a lengthy task. @@ -391,7 +391,7 @@ public class ApplicationController { // For direct deployments use the full deployment ID, but otherwise use just the tenant and application as // the source since it's the same application, so it should have the same warnings NotificationSource source = zone.environment().isManuallyDeployed() ? - NotificationSource.from(new DeploymentId(job.application(), zone)) : NotificationSource.from(applicationId); + NotificationSource.from(deployment) : NotificationSource.from(applicationId); List warnings = Optional.ofNullable(result.prepareResponse().log) .map(logs -> logs.stream() .filter(log -> log.applicationPackage) @@ -476,6 +476,7 @@ public class ApplicationController { ZoneId zone, Version platform, Set endpoints, Optional endpointCertificateMetadata, boolean dryRun) { + DeploymentId deployment = new DeploymentId(application, zone); try { Optional dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag @@ -490,7 +491,7 @@ public class ApplicationController { .map(tenant -> ((AthenzTenant)tenant).domain()); if (zone.environment().isManuallyDeployed()) - controller.applications().applicationStore().putMeta(new DeploymentId(application, zone), + controller.applications().applicationStore().putMeta(deployment, clock.instant(), applicationPackage.metaDataZip()); @@ -502,9 +503,9 @@ public class ApplicationController { .filter(tenant-> tenant instanceof CloudTenant) .map(tenant -> ((CloudTenant) tenant).tenantSecretStores()) .orElse(List.of()); - List operatorCertificates = controller.supportAccess().activeGrantsFor(new DeploymentId(application, zone)).stream() - .map(SupportAccessGrant::certificate) - .collect(toList()); + List operatorCertificates = controller.supportAccess().activeGrantsFor(deployment).stream() + .map(SupportAccessGrant::certificate) + .collect(toList()); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(new DeploymentData(application, zone, applicationPackage.zippedContent(), platform, @@ -515,10 +516,10 @@ public class ApplicationController { return new ActivateResult(new RevisionId(applicationPackage.hash()), preparedApplication.prepareResponse(), applicationPackage.zippedContent().length); } finally { - // Even if prepare fails, a load balancer may have been provisioned. Always refresh routing policies so that - // any DNS updates can be propagated as early as possible. - if ( ! application.instance().isTester()) - controller.routing().policies().refresh(application, applicationPackage.deploymentSpec(), zone); + // Even if prepare fails, routing configuration may need to be updated + if ( ! application.instance().isTester()) { + controller.routing().of(deployment).configure(applicationPackage.deploymentSpec()); + } } } @@ -702,7 +703,7 @@ public class ApplicationController { try { configServer.deactivate(id); } finally { - controller.routing().policies().refresh(application.get().id().instance(instanceName), application.get().deploymentSpec(), zone); + controller.routing().of(id).configure(application.get().deploymentSpec()); if (zone.environment().isManuallyDeployed()) applicationStore.putMetaTombstone(id, clock.instant()); if (!zone.environment().isTest()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index c832b6672d0..4772dbeaab1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -18,7 +18,6 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -31,10 +30,16 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; -import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.ExclusiveDeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext.SharedDeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.ExclusiveRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.SharedRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.rotation.RotationRepository; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import java.nio.charset.StandardCharsets; @@ -44,7 +49,6 @@ import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; -import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -77,6 +81,25 @@ public class RoutingController { this.hideSharedRoutingEndpoint = Flags.HIDE_SHARED_ROUTING_ENDPOINT.bindTo(controller.flagSource()); } + /** Create a routing context for given deployment */ + public DeploymentRoutingContext of(DeploymentId deployment) { + if (usesSharedRouting(deployment.zoneId())) { + return new SharedDeploymentRoutingContext(deployment, + this, + controller.serviceRegistry().configServer(), + controller.clock()); + } + return new ExclusiveDeploymentRoutingContext(deployment, this); + } + + /** Create a routing context for given zone */ + public RoutingContext of(ZoneId zone) { + if (usesSharedRouting(zone)) { + return new SharedRoutingContext(zone, controller.serviceRegistry().configServer()); + } + return new ExclusiveRoutingContext(zone, routingPolicies); + } + public RoutingPolicies policies() { return routingPolicies; } @@ -217,43 +240,6 @@ public class RoutingController { return Collections.unmodifiableList(endpointDnsNames); } - /** Change status of all global endpoints for given deployment */ - public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { - readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { - try { - controller.serviceRegistry().configServer().setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), status); - } catch (Exception e) { - throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); - } - }); - } - - /** Get global endpoint status for given deployment */ - public Map globalRotationStatus(DeploymentId deployment) { - var routingEndpoints = new LinkedHashMap(); - readDeclaredEndpointsOf(deployment.applicationId()).requiresRotation().primary().ifPresent(endpoint -> { - var upstreamName = endpoint.upstreamIdOf(deployment); - var status = controller.serviceRegistry().configServer().getGlobalRotationStatus(deployment, upstreamName); - routingEndpoints.put(endpoint, status); - }); - return Collections.unmodifiableMap(routingEndpoints); - } - - /** - * Assigns one or more global rotations to given application, if eligible. The given application is implicitly - * stored, ensuring that the assigned rotation(s) are persisted when this returns. - */ - private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { - try (RotationLock rotationLock = rotationRepository.lock()) { - var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), - application.get().require(instanceName), - rotationLock); - application = application.with(instanceName, instance -> instance.with(rotations)); - controller.applications().store(application); // store assigned rotation even if deployment fails - } - return application; - } - /** Returns the global and application-level endpoints for given deployment, as container endpoints */ public Set containerEndpointsOf(LockedApplication application, InstanceName instanceName, ZoneId zone) { // Assign rotations to application @@ -355,6 +341,32 @@ public class RoutingController { Priority.normal)); } + /** Returns direct routing endpoints if any exist and feature flag is set for given application */ + // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant + public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) { + boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); + EndpointList directEndpoints = endpoints.direct(); + if (hideSharedEndpoint && !directEndpoints.isEmpty()) { + return directEndpoints; + } + return endpoints; + } + + /** + * Assigns one or more global rotations to given application, if eligible. The given application is implicitly + * stored, ensuring that the assigned rotation(s) are persisted when this returns. + */ + private LockedApplication assignRotations(LockedApplication application, InstanceName instanceName) { + try (RotationLock rotationLock = rotationRepository.lock()) { + var rotations = rotationRepository.getOrAssignRotations(application.get().deploymentSpec(), + application.get().require(instanceName), + rotationLock); + application = application.with(instanceName, instance -> instance.with(rotations)); + controller.applications().store(application); // store assigned rotation even if deployment fails + } + return application; + } + private boolean usesSharedRouting(ZoneId zone) { return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); } @@ -442,23 +454,12 @@ public class RoutingController { } /** Create a common name based on a hash of given application. This must be less than 64 characters long. */ - private String commonNameHashOf(ApplicationId application, SystemName system) { + private static String commonNameHashOf(ApplicationId application, SystemName system) { HashCode sha1 = Hashing.sha1().hashString(application.serializedForm(), StandardCharsets.UTF_8); String base32 = BaseEncoding.base32().omitPadding().lowerCase().encode(sha1.asBytes()); return 'v' + base32 + Endpoint.internalDnsSuffix(system); } - /** Returns direct routing endpoints if any exist and feature flag is set for given application */ - // TODO: Remove this when feature flag is removed, and in-line .direct() filter where relevant - public EndpointList directEndpoints(EndpointList endpoints, ApplicationId application) { - boolean hideSharedEndpoint = hideSharedRoutingEndpoint.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value(); - EndpointList directEndpoints = endpoints.direct(); - if (hideSharedEndpoint && !directEndpoints.isEmpty()) { - return directEndpoints; - } - return endpoints; - } - private static String asString(Endpoint.Scope scope) { switch (scope) { case application: return "application"; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java index 94f6cccb3a5..9789bbd4da2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java @@ -41,15 +41,16 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterId; import com.yahoo.vespa.hosted.controller.api.integration.organization.DeploymentFailureMails; import com.yahoo.vespa.hosted.controller.api.integration.organization.Mail; import com.yahoo.vespa.hosted.controller.application.ActivateResult; -import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.config.ControllerConfig; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; -import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.yolean.Exceptions; import javax.security.auth.x500.X500Principal; @@ -477,12 +478,12 @@ public class InternalStepRunner implements StepRunner { } private boolean endpointsAvailable(ApplicationId id, ZoneId zone, DualLogger logger) { - var endpoints = controller.routing().readZoneEndpointsOf(Set.of(new DeploymentId(id, zone))); + DeploymentId deployment = new DeploymentId(id, zone); + Map> endpoints = controller.routing().readZoneEndpointsOf(Set.of(deployment)); if ( ! endpoints.containsKey(zone)) { logger.log("Endpoints not yet ready."); return false; } - var policies = controller.routing().policies().get(new DeploymentId(id, zone)); for (var endpoint : endpoints.get(zone)) { HostName endpointName = HostName.from(endpoint.dnsName()); var ipAddress = controller.jobController().cloud().resolveHostName(endpointName); @@ -490,10 +491,10 @@ public class InternalStepRunner implements StepRunner { logger.log(INFO, "DNS lookup yielded no IP address for '" + endpointName + "'."); return false; } - if (endpoint.routingMethod() == RoutingMethod.exclusive) { - var policy = policies.get(new RoutingPolicyId(id, ClusterSpec.Id.from(endpoint.name()), zone)); - if (policy == null) - throw new IllegalStateException(endpoint + " has no matching policy in " + policies); + DeploymentRoutingContext context = controller.routing().of(deployment); + if (context.routingMethod() == RoutingMethod.exclusive) { + RoutingPolicy policy = context.routingPolicy(ClusterSpec.Id.from(endpoint.name())) + .orElseThrow(() -> new IllegalStateException(endpoint + " has no matching policy")); var cNameValue = controller.jobController().cloud().resolveCname(endpointName); if ( ! cNameValue.map(policy.canonicalName()::equals).orElse(false)) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java index 1d5d444a32c..5acb21917eb 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/SystemRoutingPolicyMaintainer.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; @@ -25,7 +26,8 @@ public class SystemRoutingPolicyMaintainer extends ControllerMaintainer { for (var zone : controller().zoneRegistry().zones().reachable().ids()) { for (var application : SystemApplication.values()) { if (!application.hasEndpoint()) continue; - controller().routing().policies().refresh(application.id(), DeploymentSpec.empty, zone); + DeploymentId deployment = new DeploymentId(application.id(), zone); + controller().routing().of(deployment).configure(DeploymentSpec.empty); } } return 1.0; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 5d4b45fa82b..19fc155b1ca 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -44,7 +44,6 @@ import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.LockedTenant; import com.yahoo.vespa.hosted.controller.NotExistsException; import com.yahoo.vespa.hosted.controller.api.application.v4.EnvironmentResource; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.application.v4.model.ProtonMetrics; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction; import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RestartAction; @@ -101,6 +100,7 @@ import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationState; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationStatus; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.security.AccessControlRequests; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccess; @@ -1532,49 +1532,32 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (deployment == null) { throw new NotExistsException(instance + " has no deployment in " + zone); } - - // The order here matters because setGlobalRotationStatus involves an external request that may fail. - // TODO(mpolden): Set only one of these when only one kind of global endpoints are supported per zone. - var deploymentId = new DeploymentId(instance.id(), zone); - setGlobalRotationStatus(deploymentId, inService, request); - setGlobalEndpointStatus(deploymentId, inService, request); - + DeploymentId deploymentId = new DeploymentId(instance.id(), zone); + RoutingStatus.Agent agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; + RoutingStatus.Value status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out; + controller.routing().of(deploymentId).setRoutingStatus(status, agent); return new MessageResponse(Text.format("Successfully set %s in %s %s service", instance.id().toShortString(), zone, inService ? "in" : "out of")); } - /** Set the global endpoint status for given deployment. This only applies to global endpoints backed by a cloud service */ - private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) { - var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; - var status = inService ? RoutingStatus.Value.in : RoutingStatus.Value.out; - controller.routing().policies().setRoutingStatus(deployment, status, agent); - } - - /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */ - private void setGlobalRotationStatus(DeploymentId deployment, boolean inService, HttpRequest request) { - var requestData = toSlime(request.getData()).get(); - var reason = mandatory("reason", requestData).asString(); - var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; - long timestamp = controller.clock().instant().getEpochSecond(); - var status = inService ? EndpointStatus.Status.in : EndpointStatus.Status.out; - var endpointStatus = new EndpointStatus(status, reason, agent.name(), timestamp); - controller.routing().setGlobalRotationStatus(deployment, endpointStatus); - } - private HttpResponse getGlobalRotationOverride(String tenantName, String applicationName, String instanceName, String environment, String region) { DeploymentId deploymentId = new DeploymentId(ApplicationId.from(tenantName, applicationName, instanceName), requireZone(environment, region)); Slime slime = new Slime(); Cursor array = slime.setObject().setArray("globalrotationoverride"); - controller.routing().globalRotationStatus(deploymentId) - .forEach((endpoint, status) -> { - array.addString(endpoint.upstreamIdOf(deploymentId)); - Cursor statusObject = array.addObject(); - statusObject.setString("status", status.getStatus().name()); - statusObject.setString("reason", status.getReason() == null ? "" : status.getReason()); - statusObject.setString("agent", status.getAgent() == null ? "" : status.getAgent()); - statusObject.setLong("timestamp", status.getEpoch()); - }); + Optional primaryEndpoint = controller.routing().readDeclaredEndpointsOf(deploymentId.applicationId()) + .requiresRotation() + .primary(); + if (primaryEndpoint.isPresent()) { + DeploymentRoutingContext context = controller.routing().of(deploymentId); + RoutingStatus status = context.routingStatus(); + array.addString(primaryEndpoint.get().upstreamIdOf(deploymentId)); + Cursor statusObject = array.addObject(); + statusObject.setString("status", status.value().name()); + statusObject.setString("reason", ""); + statusObject.setString("agent", status.agent().name()); + statusObject.setLong("timestamp", status.changedAt().getEpochSecond()); + } return new SlimeJsonResponse(slime); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 45abf7f2946..226a7ca9561 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -19,26 +19,26 @@ import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.Instance; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.role.Role; import com.yahoo.vespa.hosted.controller.api.role.RoleDefinition; import com.yahoo.vespa.hosted.controller.api.role.SecurityContext; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLoggingRequestHandler; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; +import com.yahoo.vespa.hosted.controller.routing.context.RoutingContext; import com.yahoo.yolean.Exceptions; import java.net.URI; -import java.time.Instant; import java.util.Comparator; import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.logging.Level; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * This implements the /routing/v1 API, which provides operators and tenants routing control at both zone- (operator @@ -112,11 +112,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var deploymentsStatus = deployments.stream() .collect(Collectors.toMap( deploymentId -> deploymentId, - deploymentId -> Stream.concat( - directGlobalRoutingStatus(deploymentId).stream(), - sharedGlobalRoutingStatus(deploymentId).stream() - ).collect(Collectors.toList()) - )); + deploymentId -> controller.routing().of(deploymentId).routingStatus()) + ); var slime = new Slime(); var root = slime.setObject(); @@ -125,11 +122,11 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var endpointRoot = endpointsRoot.addObject(); endpointToSlime(endpointRoot, endpoint); var zonesRoot = endpointRoot.setArray("zones"); - endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value())).forEach(deployment -> { - deploymentsStatus.getOrDefault(deployment, List.of()).forEach(status -> { - deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod()); - }); - }); + endpoint.deployments().stream().sorted(Comparator.comparing(d -> d.zoneId().value())) + .forEach(deployment -> { + RoutingStatus status = deploymentsStatus.get(deployment); + deploymentStatusToSlime(zonesRoot.addObject(), deployment, status, endpoint.routingMethod()); + }); }); return new SlimeJsonResponse(slime); @@ -211,13 +208,10 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private HttpResponse setZoneStatus(Path path, boolean in) { - var zone = zoneFrom(path); - if (exclusiveRoutingIn(zone)) { - var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; - controller.routing().policies().setRoutingStatus(zone, status); - } else { - controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); - } + ZoneId zone = zoneFrom(path); + RoutingContext context = controller.routing().of(zone); + RoutingStatus.Value newStatus = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; + context.setRoutingStatus(newStatus, RoutingStatus.Agent.operator); return new MessageResponse("Set global routing status for deployments in " + zone + " to " + (in ? "IN" : "OUT")); } @@ -231,16 +225,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } private void toSlime(ZoneId zone, Cursor zoneObject) { - if (exclusiveRoutingIn(zone)) { - var zonePolicy = controller.routing().policies().get(zone); - zoneStatusToSlime(zoneObject, zonePolicy.zone(), zonePolicy.routingStatus(), RoutingMethod.exclusive); - } else { - // Rotation status per zone only exposes in/out status, no agent or time of change. - var in = controller.serviceRegistry().configServer().getGlobalRotationStatus(zone); - var globalRouting = new RoutingStatus(in ? RoutingStatus.Value.in : RoutingStatus.Value.out, - RoutingStatus.Agent.operator, Instant.EPOCH); - zoneStatusToSlime(zoneObject, zone, globalRouting, RoutingMethod.shared); - } + RoutingContext context = controller.routing().of(zone); + zoneStatusToSlime(zoneObject, zone, context.routingStatus(), context.routingMethod()); } private HttpResponse setDeploymentStatus(Path path, boolean in, HttpRequest request) { @@ -249,18 +235,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var status = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; var agent = isOperator(request) ? RoutingStatus.Agent.operator : RoutingStatus.Agent.tenant; requireDeployment(deployment, instance); - - if (sharedRoutingIn(deployment.zoneId())) { - // Set rotation status - var endpointStatus = new EndpointStatus(in ? EndpointStatus.Status.in : EndpointStatus.Status.out, - "", - agent.name(), - controller.clock().instant().getEpochSecond()); - controller.routing().setGlobalRotationStatus(deployment, endpointStatus); - } else { - // Set policy status - controller.routing().policies().setRoutingStatus(deployment, status, agent); - } + controller.routing().of(deployment).setRoutingStatus(status, agent); return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } @@ -279,66 +254,24 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var instances = instanceId == null ? application.instances().values() : List.of(application.instances().get(instanceId.instance())); + EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application); for (var instance : instances) { var zones = zoneId == null ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)) .collect(Collectors.toList()) : List.of(zoneId); for (var zone : zones) { - var deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance); - // Include status from rotation - sharedGlobalRoutingStatus(deploymentId).ifPresent(status -> { - deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.shared); - }); - - // Include status from routing policies - directGlobalRoutingStatus(deploymentId).forEach(status -> { - deploymentStatusToSlime(deploymentsArray.addObject(), deploymentId, status, RoutingMethod.exclusive); - }); - } - } - } - - } - - private Optional sharedGlobalRoutingStatus(DeploymentId deploymentId) { - if (sharedRoutingIn(deploymentId.zoneId())) { - var rotationStatus = controller.routing().globalRotationStatus(deploymentId); - // Status is equal across all global endpoints, as the status is per deployment, not per endpoint. - var endpointStatus = rotationStatus.values().stream().findFirst(); - if (endpointStatus.isPresent()) { - var changedAt = Instant.ofEpochSecond(endpointStatus.get().getEpoch()); - RoutingStatus.Agent agent; - try { - agent = RoutingStatus.Agent.valueOf(endpointStatus.get().getAgent()); - } catch (IllegalArgumentException e) { - agent = RoutingStatus.Agent.unknown; + DeploymentId deploymentId = requireDeployment(new DeploymentId(instance.id(), zone), instance); + DeploymentRoutingContext context = controller.routing().of(deploymentId); + if (declaredEndpoints.targets(deploymentId).isEmpty()) continue; // No declared endpoints point to this deployment + deploymentStatusToSlime(deploymentsArray.addObject(), + deploymentId, + context.routingStatus(), + context.routingMethod()); } - var status = endpointStatus.get().getStatus() == EndpointStatus.Status.in - ? RoutingStatus.Value.in - : RoutingStatus.Value.out; - return Optional.of(new RoutingStatus(status, agent, changedAt)); } } - return Optional.empty(); - } - - private List directGlobalRoutingStatus(DeploymentId deploymentId) { - return controller.routing().policies().get(deploymentId).values().stream() - .filter(p -> ! p.instanceEndpoints().isEmpty()) // This policy does not apply to a global endpoint - .filter(p -> exclusiveRoutingIn(p.id().zone())) - .map(p -> p.status().routingStatus()) - .collect(Collectors.toList()); - } - - /** Returns whether given zone uses exclusive routing */ - private boolean exclusiveRoutingIn(ZoneId zone) { - return controller.zoneRegistry().routingMethods(zone).contains(RoutingMethod.exclusive); - } - /** Returns whether given zone uses shared routing */ - private boolean sharedRoutingIn(ZoneId zone) { - return controller.zoneRegistry().routingMethods(zone).stream().anyMatch(RoutingMethod::isShared); } private static void zoneStatusToSlime(Cursor object, ZoneId zone, RoutingStatus routingStatus, RoutingMethod method) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java new file mode 100644 index 00000000000..28fbeee28f5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -0,0 +1,154 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.LockedApplication; +import com.yahoo.vespa.hosted.controller.RoutingController; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +import java.time.Clock; +import java.time.Instant; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +/** + * A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment. + * + * @author mpolden + */ +public abstract class DeploymentRoutingContext implements RoutingContext { + + final DeploymentId deployment; + final RoutingController controller; + final RoutingMethod method; + + public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController controller) { + this.deployment = Objects.requireNonNull(deployment); + this.controller = Objects.requireNonNull(controller); + this.method = Objects.requireNonNull(method); + } + + /** + * Prepare routing configuration for the deployment in this context + * + * @return the container endpoints relevant for this deployment, as declared in deployment spec + */ + public final Set prepare(LockedApplication application) { + return controller.containerEndpointsOf(application, deployment.applicationId().instance(), deployment.zoneId()); + } + + /** Configure routing for the deployment in this context, using given deployment spec */ + public final void configure(DeploymentSpec deploymentSpec) { + controller.policies().refresh(deployment.applicationId(), deploymentSpec, deployment.zoneId()); + } + + /** Routing method of this context */ + public final RoutingMethod routingMethod() { + return method; + } + + /** Read the routing policy for given cluster in this deployment */ + public final Optional routingPolicy(ClusterSpec.Id cluster) { + RoutingPolicyId id = new RoutingPolicyId(deployment.applicationId(), cluster, deployment.zoneId()); + return Optional.ofNullable(controller.policies().get(deployment).get(id)); + } + + /** + * Extension of a {@link DeploymentRoutingContext} for deployments using either {@link RoutingMethod#shared} or + * {@link RoutingMethod#sharedLayer4} routing. + */ + public static class SharedDeploymentRoutingContext extends DeploymentRoutingContext { + + private final Clock clock; + private final ConfigServer configServer; + + public SharedDeploymentRoutingContext(DeploymentId deployment, RoutingController controller, ConfigServer configServer, Clock clock) { + super(deployment, RoutingMethod.shared, controller); + this.clock = Objects.requireNonNull(clock); + this.configServer = Objects.requireNonNull(configServer); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + EndpointStatus newStatus = new EndpointStatus(value == RoutingStatus.Value.in + ? EndpointStatus.Status.in + : EndpointStatus.Status.out, + "", + agent.name(), + clock.instant().getEpochSecond()); + primaryEndpoint().ifPresent(endpoint -> { + try { + configServer.setGlobalRotationStatus(deployment, endpoint.upstreamIdOf(deployment), newStatus); + } catch (Exception e) { + throw new RuntimeException("Failed to set rotation status of " + endpoint + " in " + deployment, e); + } + }); + } + + @Override + public RoutingStatus routingStatus() { + Optional status = primaryEndpoint().map(endpoint -> { + var upstreamName = endpoint.upstreamIdOf(deployment); + return configServer.getGlobalRotationStatus(deployment, upstreamName); + }); + if (status.isEmpty()) return RoutingStatus.DEFAULT; + RoutingStatus.Agent agent; + try { + agent = RoutingStatus.Agent.valueOf(status.get().getAgent().toLowerCase()); + } catch (IllegalArgumentException e) { + agent = RoutingStatus.Agent.unknown; + } + return new RoutingStatus(status.get().getStatus() == EndpointStatus.Status.in + ? RoutingStatus.Value.in + : RoutingStatus.Value.out, + agent, + Instant.ofEpochSecond(status.get().getEpoch())); + } + + private Optional primaryEndpoint() { + return controller.readDeclaredEndpointsOf(deployment.applicationId()) + .requiresRotation() + .primary(); + } + + } + + /** + * Implementation of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#exclusive} + * routing. + */ + public static class ExclusiveDeploymentRoutingContext extends DeploymentRoutingContext { + + public ExclusiveDeploymentRoutingContext(DeploymentId deployment, RoutingController controller) { + super(deployment, RoutingMethod.exclusive, controller); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + controller.policies().setRoutingStatus(deployment, value, agent); + } + + @Override + public RoutingStatus routingStatus() { + // Status for a deployment applies to all clusters within the deployment, so we use the status from the + // first matching policy here + return controller.policies().get(deployment).values().stream() + .findFirst() + .map(RoutingPolicy::status) + .map(RoutingPolicy.Status::routingStatus) + .orElse(RoutingStatus.DEFAULT); + } + + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java new file mode 100644 index 00000000000..e949c45f2fd --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/ExclusiveRoutingContext.java @@ -0,0 +1,41 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +import java.util.Objects; + +/** + * An implementation of {@link RoutingContext} for a zone using {@link RoutingMethod#exclusive} routing. + * + * @author mpolden + */ +public class ExclusiveRoutingContext implements RoutingContext { + + private final RoutingPolicies policies; + private final ZoneId zone; + + public ExclusiveRoutingContext(ZoneId zone, RoutingPolicies policies) { + this.policies = Objects.requireNonNull(policies); + this.zone = Objects.requireNonNull(zone); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + policies.setRoutingStatus(zone, value); + } + + @Override + public RoutingStatus routingStatus() { + return policies.get(zone).routingStatus(); + } + + @Override + public RoutingMethod routingMethod() { + return RoutingMethod.exclusive; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java new file mode 100644 index 00000000000..6f43416b9b5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/RoutingContext.java @@ -0,0 +1,23 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +/** + * Top-level interface for a routing context, which provides control of routing status for a deployment or zone. + * + * @author mpolden + */ +public interface RoutingContext { + + /** Change the routing status for the zone or deployment represented by this context */ + void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent); + + /** Get the current routing status for the zone or deployment represented by this context */ + RoutingStatus routingStatus(); + + /** Routing method used in this context */ + RoutingMethod routingMethod(); + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java new file mode 100644 index 00000000000..e38212d7f80 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/SharedRoutingContext.java @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing.context; + +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; + +import java.time.Instant; +import java.util.Objects; + +/** + * An implementation of {@link RoutingContext} for a zone, using either {@link RoutingMethod#shared} or + * {@link RoutingMethod#sharedLayer4} routing. + * + * @author mpolden + */ +public class SharedRoutingContext implements RoutingContext { + + private final ConfigServer configServer; + private final ZoneId zone; + + public SharedRoutingContext(ZoneId zone, ConfigServer configServer) { + this.configServer = Objects.requireNonNull(configServer); + this.zone = Objects.requireNonNull(zone); + } + + @Override + public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { + boolean in = value == RoutingStatus.Value.in; + configServer.setGlobalRotationStatus(zone, in); + } + + @Override + public RoutingStatus routingStatus() { + boolean in = configServer.getGlobalRotationStatus(zone); + RoutingStatus.Value newValue = in ? RoutingStatus.Value.in : RoutingStatus.Value.out; + return new RoutingStatus(newValue, + RoutingStatus.Agent.operator, + Instant.EPOCH); // API does not support time of change + } + + @Override + public RoutingMethod routingMethod() { + return RoutingMethod.shared; + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 132e6caa3ca..30cdd1b8466 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -19,7 +19,6 @@ import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; -import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; @@ -40,6 +39,8 @@ import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationId; import com.yahoo.vespa.hosted.controller.routing.rotation.RotationLock; +import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; @@ -214,22 +215,18 @@ public class ControllerTest { // Check initial rotation status var deployment1 = context.deploymentIdIn(zone1); - var status1 = tester.controller().routing().globalRotationStatus(deployment1); - assertEquals(1, status1.size()); - assertTrue("All upstreams are in", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); + DeploymentRoutingContext routingContext = tester.controller().routing().of(deployment1); + RoutingStatus status1 = routingContext.routingStatus(); + assertEquals(RoutingStatus.Value.in, status1.value()); // Set the deployment out of service in the global rotation - var newStatus = new EndpointStatus(EndpointStatus.Status.out, "unit-test", ControllerTest.class.getSimpleName(), tester.clock().instant().getEpochSecond()); - tester.controller().routing().setGlobalRotationStatus(deployment1, newStatus); - status1 = tester.controller().routing().globalRotationStatus(deployment1); - assertEquals(1, status1.size()); - assertTrue("All upstreams are out", status1.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); - assertTrue("Reason is set", status1.values().stream().allMatch(es -> es.getReason().equals("unit-test"))); + routingContext.setRoutingStatus(RoutingStatus.Value.out, RoutingStatus.Agent.operator); + RoutingStatus status2 = routingContext.routingStatus(); + assertEquals(RoutingStatus.Value.out, status2.value()); // Other deployment remains in - var status2 = tester.controller().routing().globalRotationStatus(context.deploymentIdIn(zone2)); - assertEquals(1, status2.size()); - assertTrue("All upstreams are in", status2.values().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.in)); + RoutingStatus status3 = tester.controller().routing().of(context.deploymentIdIn(zone2)).routingStatus(); + assertEquals(RoutingStatus.Value.in, status3.value()); } @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index ae6232ae419..afd67824ee8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -69,6 +69,7 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; +import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; import com.yahoo.vespa.hosted.controller.security.AthenzCredentials; import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; @@ -1868,13 +1869,12 @@ public class ApplicationApiTest extends ControllerContainerTest { } private void assertGlobalRouting(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) { - var changedAt = tester.controller().clock().instant(); - var westPolicies = tester.controller().routing().policies().get(deployment); - assertEquals(1, westPolicies.size()); - var westPolicy = westPolicies.values().iterator().next(); - assertEquals(value, westPolicy.status().routingStatus().value()); - assertEquals(agent, westPolicy.status().routingStatus().agent()); - assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().routingStatus().changedAt()); + Instant changedAt = tester.controller().clock().instant(); + DeploymentRoutingContext context = tester.controller().routing().of(deployment); + RoutingStatus status = context.routingStatus(); + assertEquals(value, status.value()); + assertEquals(agent, status.agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.SECONDS), status.changedAt()); } private static class RequestBuilder implements Supplier { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json index 934e0cf43b9..de2266fd197 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/global-rotation-get.json @@ -4,7 +4,7 @@ { "status": "in", "reason": "", - "agent": "", + "agent": "unknown", "timestamp": 1497618757 } ] -- cgit v1.2.3