diff options
15 files changed, 194 insertions, 88 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index c44c533e995..f28c877dce6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -251,7 +251,6 @@ public class ApplicationController { /** Change status of all global endpoints for given deployment */ public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) { var globalEndpoints = findGlobalEndpoints(deployment); - if (globalEndpoints.isEmpty()) throw new IllegalArgumentException(deployment + " has no global endpoints"); globalEndpoints.forEach(endpoint -> { try { configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(), status); @@ -620,10 +619,10 @@ public class ApplicationController { throw new NotExistsException("Deployment", deploymentId.toString()); try { - return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream() - .map(RoutingEndpoint::endpoint) - .map(URI::create) - .iterator()); + return findRoutingEndpoints(deploymentId).stream() + .map(RoutingEndpoint::endpoint) + .map(URI::create) + .collect(Collectors.toUnmodifiableList()); } catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e); @@ -631,6 +630,21 @@ public class ApplicationController { } } + /** Find the routing endpoints of a given deployment. Routing endpoints are owned by the shared routing layer. */ + private List<RoutingEndpoint> findRoutingEndpoints(DeploymentId deployment) { + if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) { + return List.of(); // No shared routing layer in this zone. + } + return routingGenerator.endpoints(deployment); + } + + private Map<ClusterSpec.Id, URI> findClusterEndpoints(DeploymentId deployment) { + if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) { + return Map.of(); // No shared routing layer in this zone. + } + return routingGenerator.clusterEndpoints(deployment); + } + /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */ public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) { if ( ! getInstance(id.applicationId()) @@ -640,7 +654,7 @@ public class ApplicationController { // TODO(jvenstad): Swap to use routingPolicies first, when this is ready. try { - var endpoints = routingGenerator.clusterEndpoints(id); + var endpoints = findClusterEndpoints(id); if ( ! endpoints.isEmpty()) return endpoints; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index 6d7980396de..13b3b368293 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableSortedMap; import com.yahoo.component.Version; +import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; @@ -506,12 +507,17 @@ public class JobController { }); } + // TODO(mpolden): Eliminate duplication in this and ApplicationController#deactivate public void deactivateTester(TesterId id, JobType type) { + var zone = type.zone(controller.system()); try { - controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system()))); - } - catch (NotFoundException ignored) { + controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), zone)); + } catch (NotFoundException ignored) { // Already gone -- great! + } finally { + // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and + // tester instances have none. + controller.applications().routingPolicies().refresh(id.id(), DeploymentSpec.empty, zone); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java index 4c7305f08e5..ad8c0c73754 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java @@ -30,6 +30,9 @@ import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.Stream; +/** + * @author andreer + */ public class EndpointCertificateManager { private static final Logger log = Logger.getLogger(EndpointCertificateManager.class.getName()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index cad206b0a69..be459614880 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -83,8 +83,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { } else { controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in); } - return new MessageResponse("Set global routing status for deployments in " + zone + " to '" + - (in ? "in" : "out") + "'"); + return new MessageResponse("Set global routing status for deployments in " + zone + " to " + + (in ? "IN" : "OUT")); } private HttpResponse zoneStatus(Path path) { @@ -120,7 +120,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { // Set policy status controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent); - return new MessageResponse("Set global routing status for " + deployment + " to '" + (in ? "in" : "out") + "'"); + return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT")); } private HttpResponse deploymentStatus(Path path) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 88f746a9c51..961a3674471 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -84,10 +84,10 @@ public class RoutingPolicies { deploymentSpec); var inactiveZones = inactiveZones(application, deploymentSpec); try (var lock = db.lockRoutingPolicies()) { - removeGlobalDnsUnreferencedBy(loadBalancers, lock); + if (!application.instance().isTester()) removeGlobalDnsUnreferencedBy(loadBalancers, lock); storePoliciesOf(loadBalancers, lock); removePoliciesUnreferencedBy(loadBalancers, lock); - updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); + if (!application.instance().isTester()) updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 02c86f78ec0..83f80488ee5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -210,17 +210,6 @@ public class ControllerTest { assertEquals(2, upstreamOneEndpoints.get().size()); assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out)); assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test"))); - - // Deployment without a global endpoint - tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of( - new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"), - new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"), - new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3") - )); - try { - tester.controller().applications().setGlobalRotationStatus(deployment, status); - fail("Expected exception"); - } catch (IllegalArgumentException ignored) {} } @Test @@ -715,8 +704,10 @@ public class ControllerTest { // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); - var applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod) - .region("us-west-1") + var prodZone = ZoneId.from("prod", "us-west-1"); + tester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(prodZone)); + var applicationPackage = new ApplicationPackageBuilder().environment(prodZone.environment()) + .region(prodZone.region()) .build(); // Deploy app1 in production context1.submit(applicationPackage).deploy(); @@ -725,7 +716,7 @@ public class ControllerTest { assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", "app1.tenant1.global.vespa.oath.cloud", "*.app1.tenant1.global.vespa.oath.cloud"), - tester.controller().zoneRegistry().zones().all().ids().stream() + tester.controller().zoneRegistry().zones().directlyRouted().ids().stream() .flatMap(zone -> Stream.of("", "*.") .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + @@ -739,13 +730,13 @@ public class ControllerTest { // Create app2 var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); - ZoneId zone = ZoneId.from("dev", "us-east-1"); + var devZone = ZoneId.from("dev", "us-east-1"); // Deploy app2, after "removing" direct routing everywhere tester.controllerTester().zoneRegistry().setDirectlyRouted(); - tester.controller().applications().deploy(context2.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none()); + tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none()); assertTrue("Application deployed and activated", - tester.configServer().application(context2.instanceId(), zone).get().activated()); + tester.configServer().application(context2.instanceId(), devZone).get().activated()); assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent()); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index ebf64f90873..c90abe5ed5c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; @@ -17,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -99,6 +101,7 @@ public class DeploymentContext { private final RoutingGeneratorMock routing; private final JobRunner runner; private final DeploymentTester tester; + private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>(); private ApplicationVersion lastSubmission = null; private boolean deferDnsUpdates = false; @@ -190,6 +193,15 @@ public class DeploymentContext { return this; } + /** + * Defer provisioning of load balancers in zones in given environment. Default behaviour is to automatically + * provision load balancers in all zones that support direct routing. + */ + public DeploymentContext deferLoadBalancerProvisioningIn(Environment... environment) { + deferLoadBalancerProvisioning.addAll(List.of(environment)); + return this; + } + /** Defer DNS updates */ public DeploymentContext deferDnsUpdates() { @@ -438,6 +450,25 @@ public class DeploymentContext { ZoneId zone = zone(job); DeploymentId deployment = new DeploymentId(job.application(), zone); + // Provision load balancers in directly routed zones, unless explicitly deferred + if (provisionLoadBalancerIn(zone)) { + if (job.type().isTest()) { + var testerDeployment = new DeploymentId(testerId.id(), zone); + configServer().putLoadBalancers(zone, List.of(new LoadBalancer(testerDeployment.toString(), + testerDeployment.applicationId(), + ClusterSpec.Id.from("default"), + HostName.from("lb-host"), + LoadBalancer.State.active, + Optional.of("dns-zone")))); + } + configServer().putLoadBalancers(zone, List.of(new LoadBalancer(deployment.toString(), + deployment.applicationId(), + ClusterSpec.Id.from("default"), + HostName.from("lb-host"), + LoadBalancer.State.active, + Optional.of("dns-zone")))); + } + // First step is always a deployment. runner.advance(currentRun(job)); @@ -493,6 +524,7 @@ public class DeploymentContext { /** Sets a single endpoint in the routing layer; this matches that required for the tester */ private DeploymentContext setEndpoints(ZoneId zone, boolean tester) { + if (isDirectlyRouted(zone)) return this; var id = instanceId; if (tester) { id = testerId.id(); @@ -542,8 +574,12 @@ public class DeploymentContext { assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); configServer().convergeServices(TesterId.of(id.application()).id(), zone); runner.advance(currentRun(job)); - assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); - setTesterEndpoints(zone); + if (provisionLoadBalancerIn(zone)) { // Endpoints are available immediately after deployment in directly routed zones + assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester)); + } else { + assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester)); + setTesterEndpoints(zone); + } runner.advance(currentRun(job)); } @@ -577,6 +613,15 @@ public class DeploymentContext { routing.removeEndpoints(new DeploymentId(TesterId.of(job.application()).id(), zone)); } + /** Returns whether a load balancer is expected to be provisioned in given zone */ + private boolean provisionLoadBalancerIn(ZoneId zone) { + return !deferLoadBalancerProvisioning.contains(zone.environment()) && isDirectlyRouted(zone); + } + + private boolean isDirectlyRouted(ZoneId zone) { + return tester.controller().zoneRegistry().zones().directlyRouted().ids().contains(zone); + } + private JobId jobId(JobType type) { return new JobId(instanceId, type); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java index 231f8be2ed3..808d9ca05c4 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java @@ -13,15 +13,20 @@ import org.junit.Test; import java.time.Clock; import java.util.Optional; +import java.util.Set; import static org.junit.Assert.assertTrue; +/** + * @author andreer + */ public class EndpointCertificateManagerTest { @Test public void getEndpointCertificate() { SecretStoreMock secretStore = new SecretStoreMock(); ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main); + zoneRegistryMock.setDirectlyRouted(Set.copyOf(zoneRegistryMock.zones().all().zones())); MockCuratorDb mockCuratorDb = new MockCuratorDb(); ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock(); Clock clock = Clock.systemUTC(); @@ -31,4 +36,5 @@ public class EndpointCertificateManagerTest { Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, id); assertTrue(endpointCertificateMetadata.isPresent()); } -}
\ No newline at end of file + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 66c7334dc2d..eb8af62a0e6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -44,6 +44,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -69,7 +70,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>(); private final Version initialVersion = new Version(6, 1, 0); private final Set<DeploymentId> suspendedApplications = new HashSet<>(); - private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>(); + private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>(); private final Map<DeploymentId, List<Log>> warnings = new HashMap<>(); private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>(); private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>(); @@ -260,8 +261,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer disallowConvergenceCheckApplications.add(applicationId); } - private List<LoadBalancer> getLoadBalancers(ZoneId zone) { - return loadBalancers.getOrDefault(zone, Collections.emptyList()); + private Set<LoadBalancer> getLoadBalancers(ZoneId zone) { + return loadBalancers.getOrDefault(zone, new LinkedHashSet<>()); } @Override @@ -281,8 +282,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer return TesterCloud.Status.SUCCESS; } - public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { - this.loadBalancers.putIfAbsent(zone, new ArrayList<>()); + /** Add any of given loadBalancers that do not already exist to the load balancers in zone */ + public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) { + this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>()); this.loadBalancers.get(zone).addAll(loadBalancers); } @@ -366,6 +368,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate"); applications.remove(deployment); serviceStatus.remove(deployment); + removeLoadBalancers(deployment.applicationId(), deployment.zoneId()); } // Returns a canned example response diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java index 269bdcc5dca..611ea4327ed 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java @@ -8,6 +8,8 @@ import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import java.util.Objects; + /** * @author hakonhall */ @@ -34,6 +36,10 @@ public class ZoneApiMock implements ZoneApi { return newBuilder().with(ZoneId.from(environment, region)).build(); } + public static ZoneApiMock from(ZoneId zone) { + return newBuilder().with(zone).build(); + } + @Override public SystemName getSystemName() { return systemName; } @@ -46,6 +52,19 @@ public class ZoneApiMock implements ZoneApi { @Override public String getCloudNativeRegionName() { return cloudNativeRegionName; } + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneApiMock that = (ZoneApiMock) o; + return id.equals(that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + public static class Builder { private SystemName systemName = SystemName.defaultSystem(); private ZoneId id = ZoneId.defaultId(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java index 6f8ff39456f..3cb7a13dd54 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java @@ -60,7 +60,6 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry ZoneApiMock.fromId("prod.us-west-1"), ZoneApiMock.fromId("prod.us-central-1"), ZoneApiMock.fromId("prod.eu-west-1"))); - setDirectlyRouted(Set.copyOf(this.zones)); } public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java index b7edcef97e4..e5670139b0f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java @@ -6,6 +6,7 @@ import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.restapi.ContainerTester; import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest; import org.junit.Before; @@ -13,7 +14,6 @@ import org.junit.Test; import java.io.File; import java.util.List; -import java.util.Set; import static org.junit.Assert.assertNotEquals; @@ -36,13 +36,17 @@ public class RoutingApiTest extends ControllerContainerTest { @Test public void policy_based_routing() { var context = deploymentTester.newDeploymentContext(); - - // Deploy application + // Zones support direct routing var westZone = ZoneId.from("prod", "us-west-1"); var eastZone = ZoneId.from("prod", "us-east-3"); + deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone), + ZoneApiMock.from(eastZone)); + + // Deploy application var applicationPackage = new ApplicationPackageBuilder() .region(westZone.region()) .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) .build(); context.submit(applicationPackage).deploy(); context.addRoutingPolicy(westZone, true); @@ -56,7 +60,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets deployment out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/deployment-status-out.json")); @@ -64,7 +68,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets deployment in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/deployment-status-in.json")); @@ -77,7 +81,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets zone out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/zone-status-out.json")); @@ -85,7 +89,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets zone in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("policy/zone-status-in.json")); @@ -93,8 +97,6 @@ public class RoutingApiTest extends ControllerContainerTest { @Test public void rotation_based_routing() { - // No zones support direct routing - deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(Set.of()); // Deploy application var context = deploymentTester.newDeploymentContext(); var westZone = ZoneId.from("prod", "us-west-1"); @@ -102,7 +104,7 @@ public class RoutingApiTest extends ControllerContainerTest { var applicationPackage = new ApplicationPackageBuilder() .region(westZone.region()) .region(eastZone.region()) - .endpoint("default", "qrs", eastZone.region().value(), westZone.region().value()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) .build(); context.submit(applicationPackage).deploy(); @@ -116,7 +118,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets deployment out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/deployment-status-out.json")); @@ -124,7 +126,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets deployment in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/deployment-status-in.json")); @@ -137,7 +139,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets zone out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/zone-status-out.json")); @@ -145,15 +147,31 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets zone in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1", "", Request.Method.GET), new File("rotation/zone-status-in.json")); + } + + // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation + @Test + public void mixed_routing() { + // Deploy application + var context = deploymentTester.newDeploymentContext(); + var westZone = ZoneId.from("prod", "us-west-1"); + var eastZone = ZoneId.from("prod", "us-east-3"); + var applicationPackage = new ApplicationPackageBuilder() + .region(westZone.region()) + .region(eastZone.region()) + .endpoint("default", "default", eastZone.region().value(), westZone.region().value()) + .build(); + context.submit(applicationPackage).deploy(); - // TODO(mpolden): Remove the following once a zone supports either of routing policy and rotation + // Assign policy in one zone + deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone)); + context.addRoutingPolicy(westZone, true); // GET status with both policy and rotation assigned - context.addRoutingPolicy(westZone, true); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("multi-status-initial.json")); @@ -161,7 +179,7 @@ public class RoutingApiTest extends ControllerContainerTest { // POST sets deployment out tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.POST), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("multi-status-out.json")); @@ -169,7 +187,7 @@ public class RoutingApiTest extends ControllerContainerTest { // DELETE sets deployment in tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.DELETE), - "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}"); + "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}"); tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1", "", Request.Method.GET), new File("multi-status-in.json")); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json index a15a0cc8a99..cdf3e093f97 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json @@ -6,7 +6,7 @@ "environment": "prod", "region": "us-west-1", "status": "in", - "agent": "operator", + "agent": "unknown", "changedAt": "(ignore)" }, { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java index 22ed079b501..1eb4523c344 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java @@ -111,8 +111,6 @@ public class SystemFlagsDeployerTest { SystemFlagsDeployResult result = deployer.deployFlags(archive, false); - System.out.println(result); - assertThat(result.errors()).containsOnly( OperationError.listFailed(exception.getMessage(), prodUsWest1Target)); assertThat(result.flagChanges()).containsOnly( @@ -123,4 +121,4 @@ public class SystemFlagsDeployerTest { return FlagData.deserializeUtf8Json(Files.readAllBytes(Paths.get("src/test/resources/system-flags/" + filename))); } -}
\ No newline at end of file +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index ec169633433..c6f9bb81b63 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,10 +1,12 @@ // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; +import com.google.common.collect.Sets; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; @@ -30,7 +32,6 @@ import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -72,7 +73,7 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); // Creates alias records - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); @@ -91,7 +92,7 @@ public class RoutingPoliciesTest { .build(); numberOfDeployments++; tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); - context1.submit(applicationPackage2).deploy(); + context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Endpoints are updated to contain cluster in new deployment tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); @@ -106,7 +107,7 @@ public class RoutingPoliciesTest { .region(zone2.region()) .endpoint("r0", "c0") .build(); - context2.submit(applicationPackage3).deploy(); + context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // All endpoints for app1 are removed @@ -116,7 +117,7 @@ public class RoutingPoliciesTest { .region(zone3.region()) .allow(ValidationId.globalEndpointChange) .build(); - context1.submit(applicationPackage4).deploy(); + context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); @@ -136,7 +137,7 @@ public class RoutingPoliciesTest { // Deploy application int clustersPerZone = 2; tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones Set<String> expectedRecords = Set.of( @@ -149,13 +150,13 @@ public class RoutingPoliciesTest { assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Next deploy does nothing - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Add 1 cluster in each zone and deploy tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -169,7 +170,7 @@ public class RoutingPoliciesTest { // Deploy another application tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2); - context2.submit(applicationPackage).deploy(); + context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -182,12 +183,12 @@ public class RoutingPoliciesTest { "c0.app2.tenant1.us-west-1.vespa.oath.cloud", "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); - assertEquals(expectedRecords, tester.recordNames()); + assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList())); assertEquals(4, tester.policiesOf(context2.instanceId()).size()); // Deploy removes cluster from app1 tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); - context1.submit(applicationPackage).deploy(); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", @@ -228,7 +229,7 @@ public class RoutingPoliciesTest { .region(zone1.region().value()) .endpoint("r0", "c0") .build(); - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; assertEquals(endpoint + " points to c0 in all regions", @@ -243,9 +244,9 @@ public class RoutingPoliciesTest { public void cluster_endpoints_resolve_from_policies() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.provisionLoadBalancers(3, context.instanceId(), zone1); - context.submit(applicationPackage).deploy(); - tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), Collections.emptyList()); + tester.provisionLoadBalancers(3, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), List.of()); assertEquals(Map.of(ClusterSpec.Id.from("c0"), URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"), ClusterSpec.Id.from("c1"), @@ -289,7 +290,8 @@ public class RoutingPoliciesTest { tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) .setDirectlyRouted(zoneApi); - + var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); + assertEquals(prodRecords, tester.recordNames()); // Deploy to dev under different instance var devInstance = context.application().id().instance("user"); @@ -300,7 +302,7 @@ public class RoutingPoliciesTest { // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(devInstance).size()); - assertEquals(Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + assertEquals(Sets.union(prodRecords, Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); } @Test @@ -315,7 +317,7 @@ public class RoutingPoliciesTest { .build(); // Application is deployed - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); var expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud" ); @@ -334,10 +336,10 @@ public class RoutingPoliciesTest { newHostname, LoadBalancer.State.active, Optional.of("dns-zone-1")); - tester.controllerTester().configServer().addLoadBalancers(zone1, List.of(loadBalancer)); + tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); // Application redeployment preserves DNS record - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); assertEquals(1, tester.policiesOf(context.instanceId()).size()); assertEquals("CNAME points to current load blancer", newHostname.value() + ".", @@ -357,7 +359,7 @@ public class RoutingPoliciesTest { .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Global DNS record is created tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); @@ -386,7 +388,7 @@ public class RoutingPoliciesTest { assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); // Next deployment does not affect status - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); @@ -411,7 +413,7 @@ public class RoutingPoliciesTest { .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); - context.submit(applicationPackage2).deploy(); + context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); @@ -422,7 +424,7 @@ public class RoutingPoliciesTest { .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); - context.submit(applicationPackage3).deploy(); + context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); } @@ -442,7 +444,7 @@ public class RoutingPoliciesTest { .build(); for (var context : contexts) { tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - context.submit(applicationPackage).deploy(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } @@ -520,12 +522,14 @@ public class RoutingPoliciesTest { public RoutingPoliciesTester(DeploymentTester tester) { this.tester = tester; + // Make all zones directly routed + tester.controllerTester().zoneRegistry().setDirectlyRouted(Set.copyOf(tester.controllerTester().zoneRegistry().zones().all().zones())); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { for (ZoneId zone : zones) { tester.configServer().removeLoadBalancers(application, zone); - tester.configServer().addLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); + tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone)); } } |