// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.routing; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Sets; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** * @author mortent * @author mpolden */ public class RoutingPoliciesTest { private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); private static final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); private static final ZoneId zone3 = ZoneId.from("prod", "aws-us-east-1a"); private static final ZoneId zone4 = ZoneId.from("prod", "aws-us-east-1b"); private static final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) .build(); @Test public void global_routing_policies() { var tester = new RoutingPoliciesTester(); var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); int clustersPerZone = 2; int numberOfDeployments = 2; var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0") .endpoint("r1", "c0", "us-west-1") .endpoint("r2", "c1") .build(); tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); // Creates alias records context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); assertEquals("Routing policy count is equal to cluster count", numberOfDeployments * clustersPerZone, tester.policiesOf(context1.instance().id()).size()); // Applications gains a new deployment ApplicationPackage applicationPackage2 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .region(zone3.region()) .endpoint("r0", "c0") .endpoint("r1", "c0", "us-west-1") .endpoint("r2", "c1") .build(); numberOfDeployments++; tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Endpoints are updated to contain cluster in new deployment tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); // Another application is deployed with a single cluster and global endpoint var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); var applicationPackage3 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0") .build(); context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // A deployment of app2 is removed var applicationPackage4 = applicationPackageBuilder() .region(zone1.region()) .endpoint("r0", "c0") .allow(ValidationId.globalEndpointChange) .allow(ValidationId.deploymentRemoval) .build(); context2.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1); assertEquals(1, tester.policiesOf(context2.instanceId()).size()); // All endpoints for app1 are removed ApplicationPackage applicationPackage5 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .region(zone3.region()) .allow(ValidationId.globalEndpointChange) .build(); context1.submit(applicationPackage5).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); var policies = tester.policiesOf(context1.instanceId()); assertEquals(clustersPerZone * numberOfDeployments, policies.size()); assertTrue("Rotation membership is removed from all policies", policies.asList().stream().allMatch(policy -> policy.instanceEndpoints().isEmpty())); assertEquals("Rotations for " + context2.application() + " are not removed", 1, tester.aliasDataOf(endpoint4).size()); } @Test public void global_routing_policies_with_duplicate_region() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); int clustersPerZone = 2; int numberOfDeployments = 3; var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone3.region()) .region(zone4.region()) .endpoint("r0", "c0") .endpoint("r1", "c1") .build(); tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone3, zone4); // Creates alias records context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone3, zone4); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 1, zone1, zone3, zone4); assertEquals("Routing policy count is equal to cluster count", numberOfDeployments * clustersPerZone, tester.policiesOf(context.instance().id()).size()); // A zone in shared region is set out tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); // Weight of inactive zone is set to zero tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L, zone3, 1L, zone4, 0L)); // Other zone in shared region is set out. Entire record group for the region is removed as all zones in the // region are out (weight sum = 0) tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L)); // Everything is set back in tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L, zone3, 1L, zone4, 1L)); } @Test public void global_routing_policies_legacy_global_service_id() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); int clustersPerZone = 2; int numberOfDeployments = 2; var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .globalServiceId("c0") .build(); tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2); // Creates alias records context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); assertEquals("Routing policy count is equal to cluster count", numberOfDeployments * clustersPerZone, tester.policiesOf(context.instance().id()).size()); } @Test public void zone_routing_policies() { zone_routing_policies(false); zone_routing_policies(true); } private void zone_routing_policies(boolean sharedRoutingLayer) { var tester = new RoutingPoliciesTester(); var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); // Deploy application int clustersPerZone = 2; tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones Set expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app1.tenant1.us-central-1.vespa.oath.cloud", "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Next deploy does nothing context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); assertEquals(4, tester.policiesOf(context1.instanceId()).size()); // Add 1 cluster in each zone and deploy tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", "c2.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app1.tenant1.us-central-1.vespa.oath.cloud", "c1.app1.tenant1.us-central-1.vespa.oath.cloud", "c2.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertEquals(6, tester.policiesOf(context1.instanceId()).size()); // Deploy another application tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2); context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", "c2.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app1.tenant1.us-central-1.vespa.oath.cloud", "c1.app1.tenant1.us-central-1.vespa.oath.cloud", "c2.app1.tenant1.us-central-1.vespa.oath.cloud", "c0.app2.tenant1.us-central-1.vespa.oath.cloud", "c1.app2.tenant1.us-central-1.vespa.oath.cloud", "c0.app2.tenant1.us-west-1.vespa.oath.cloud", "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList())); assertEquals(4, tester.policiesOf(context2.instanceId()).size()); // Deploy removes cluster from app1 tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app1.tenant1.us-central-1.vespa.oath.cloud", "c1.app1.tenant1.us-central-1.vespa.oath.cloud", "c0.app2.tenant1.us-central-1.vespa.oath.cloud", "c1.app2.tenant1.us-central-1.vespa.oath.cloud", "c0.app2.tenant1.us-west-1.vespa.oath.cloud", "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); // Remove app2 completely tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet() .forEach(zone -> { tester.controllerTester().configServer().removeLoadBalancers(context2.instanceId(), zone); tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone); }); context2.flushDnsUpdates(); expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app1.tenant1.us-central-1.vespa.oath.cloud", "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().read(context2.instanceId()).isEmpty()); assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().read(context1.instanceId()).size()); } @Test public void zone_routing_policies_without_dns_update() { var tester = new RoutingPoliciesTester(new DeploymentTester(), false); var context = tester.newDeploymentContext("tenant1", "app1", "default"); tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(0, tester.controllerTester().controller().curator().readNameServiceQueue().requests().size()); assertEquals(Set.of(), tester.recordNames()); assertEquals(2, tester.policiesOf(context.instanceId()).size()); } @Test public void global_routing_policies_in_rotationless_system() { var tester = new RoutingPoliciesTester(SystemName.Public); var context = tester.newDeploymentContext("tenant1", "app1", "default"); List prodZones = tester.controllerTester().controller().zoneRegistry().zones().all().in(Environment.prod).ids(); ZoneId zone1 = prodZones.get(0); ZoneId zone2 = prodZones.get(1); tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); var applicationPackage = applicationPackageBuilder() .region(zone1.region().value()) .endpoint("r0", "c0") .trustDefaultCertificate() .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); assertTrue("No rotations assigned", context.application().instances().values().stream() .map(Instance::rotations) .allMatch(List::isEmpty)); } @Test public void global_routing_policies_in_public() { var tester = new RoutingPoliciesTester(SystemName.Public); var context = tester.newDeploymentContext("tenant1", "app1", "default"); List prodZones = tester.controllerTester().controller().zoneRegistry().zones().all().in(Environment.prod).ids(); ZoneId zone1 = prodZones.get(0); ZoneId zone2 = prodZones.get(1); var applicationPackage = applicationPackageBuilder() .region(zone1.region().value()) .region(zone2.region().value()) .endpoint("default", "default") .trustDefaultCertificate() .build(); context.submit(applicationPackage).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), ClusterSpec.Id.from("default"), 0, Map.of(zone1, 1L, zone2, 1L)); assertEquals("Registers expected DNS names", Set.of("app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "app1.tenant1.aws-us-east-1.w.vespa-app.cloud", "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "app1.tenant1.g.vespa-app.cloud"), tester.recordNames()); } @Test public void manual_deployment_creates_routing_policy() { // Empty application package is valid in manually deployed environments var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); var emptyApplicationPackage = new ApplicationPackageBuilder().build(); var zone = ZoneId.from("dev", "us-east-1"); var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); tester.controllerTester().serviceRegistry().zoneRegistry() .setZones(zoneApi) .exclusiveRoutingIn(zoneApi); // Deploy to dev context.runJob(zone, emptyApplicationPackage); assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); context.flushDnsUpdates(); // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(context.instanceId()).size()); assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); } @Test public void manual_deployment_creates_routing_policy_with_non_empty_spec() { // Initial deployment var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); context.submit(applicationPackage).deploy(); var zone = ZoneId.from("dev", "us-east-1"); tester.controllerTester().setRoutingMethod(List.of(zone), RoutingMethod.exclusive); var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); assertEquals(prodRecords, tester.recordNames()); // Deploy to dev under different instance var devContext = tester.newDeploymentContext(context.application().id().instance("user")); devContext.runJob(zone, applicationPackage); assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); context.flushDnsUpdates(); // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(devContext.instanceId()).size()); assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); } @Test public void reprovisioning_load_balancer_preserves_cname_record() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); // Initial load balancer is provisioned tester.provisionLoadBalancers(1, context.instanceId(), zone1); var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .build(); // Application is deployed context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); var expectedRecords = Set.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertEquals(1, tester.policiesOf(context.instanceId()).size()); // Application is removed and the load balancer is deprovisioned tester.controllerTester().controller().applications().deactivate(context.instanceId(), zone1); tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1); // Load balancer for the same application is provisioned again, but with a different hostname var newHostname = HostName.of("new-hostname"); var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(), context.instanceId(), ClusterSpec.Id.from("c0"), Optional.of(newHostname), LoadBalancer.State.active, Optional.of("dns-zone-1")); tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); // Application redeployment preserves DNS record context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(expectedRecords, tester.recordNames()); assertEquals(1, tester.policiesOf(context.instanceId()).size()); assertEquals("CNAME points to current load blancer", newHostname.value() + ".", tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); } @Test public void set_global_endpoint_status() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); // Provision load balancers and deploy application tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Global DNS record is created tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); // Global routing status is overridden in one zone var changedAt = tester.controllerTester().clock().instant(); tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); // Inactive zone is removed from global DNS record tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); // Status details is stored in policy var policy1 = tester.routingPolicies().read(context.deploymentIdIn(zone1)).first().get(); assertEquals(RoutingStatus.Value.out, policy1.status().routingStatus().value()); assertEquals(RoutingStatus.Agent.tenant, policy1.status().routingStatus().agent()); assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().routingStatus().changedAt()); // Other zone remains in var policy2 = tester.routingPolicies().read(context.deploymentIdIn(zone2)).first().get(); assertEquals(RoutingStatus.Value.in, policy2.status().routingStatus().value()); assertEquals(RoutingStatus.Agent.system, policy2.status().routingStatus().agent()); assertEquals(Instant.EPOCH, policy2.status().routingStatus().changedAt()); // Next deployment does not affect status context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); // Deployment is set back in tester.controllerTester().clock().advance(Duration.ofHours(1)); changedAt = tester.controllerTester().clock().instant(); tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); policy1 = tester.routingPolicies().read(context.deploymentIdIn(zone1)).first().get(); assertEquals(RoutingStatus.Value.in, policy1.status().routingStatus().value()); assertEquals(RoutingStatus.Agent.tenant, policy1.status().routingStatus().agent()); assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().routingStatus().changedAt()); // Deployment is set out through a new deployment.xml var applicationPackage2 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region(), false) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); // ... back in var applicationPackage3 = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); } @Test public void set_zone_global_endpoint_status() { var tester = new RoutingPoliciesTester(); var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var context2 = tester.newDeploymentContext("tenant2", "app2", "default"); var contexts = List.of(context1, context2); // Deploy applications var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) .build(); for (var context : contexts) { tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } // Set zone out tester.routingPolicies().setRoutingStatus(zone2, RoutingStatus.Value.out); context1.flushDnsUpdates(); tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); for (var context : contexts) { var policies = tester.routingPolicies().read(context.instanceId()); assertTrue("Global routing status for policy remains " + RoutingStatus.Value.in, policies.asList().stream() .map(RoutingPolicy::status) .map(RoutingPolicy.Status::routingStatus) .map(RoutingStatus::value) .allMatch(status -> status == RoutingStatus.Value.in)); } var changedAt = tester.controllerTester().clock().instant(); var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); assertEquals(RoutingStatus.Value.out, zonePolicy.routingStatus().value()); assertEquals(RoutingStatus.Agent.operator, zonePolicy.routingStatus().agent()); assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.routingStatus().changedAt()); // Setting status per deployment does not affect status as entire zone is out tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context1.flushDnsUpdates(); tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); // Set single deployment out tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context1.flushDnsUpdates(); // Set zone back in. Deployment set explicitly out, remains out, the rest are in tester.routingPolicies().setRoutingStatus(zone2, RoutingStatus.Value.in); context1.flushDnsUpdates(); tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); } @Test public void non_production_deployment_is_not_registered_in_global_endpoint() { var tester = new RoutingPoliciesTester(SystemName.Public); // Configure the system to use the same region for test, staging and prod var context = tester.tester.newDeploymentContext(); var endpointId = EndpointId.of("r0"); var applicationPackage = applicationPackageBuilder() .trustDefaultCertificate() .region("aws-us-east-1c") .endpoint(endpointId.id(), "default") .build(); // Application starts deployment context = context.submit(applicationPackage); for (var testJob : List.of(DeploymentContext.systemTest, DeploymentContext.stagingTest)) { context = context.runJob(testJob); // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the // deployment, we consume only one DNS update at a time here do { context.flushDnsUpdates(1); tester.assertTargets(context.instanceId(), endpointId, 0); } while (!tester.recordNames().isEmpty()); } // Deployment completes context.completeRollout(); tester.assertTargets(context.instanceId(), endpointId, ClusterSpec.Id.from("default"), 0, Map.of(ZoneId.from("prod", "aws-us-east-1c"), 1L)); } @Test public void changing_global_routing_status_never_removes_all_members() { var tester = new RoutingPoliciesTester(); var context = tester.newDeploymentContext("tenant1", "app1", "default"); // Provision load balancers and deploy application tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); var applicationPackage = applicationPackageBuilder() .region(zone1.region()) .region(zone2.region()) .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Global DNS record is created, pointing to all configured zones tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // Global routing status is overridden for one deployment tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); // Setting other deployment out implicitly sets all deployments in. Weight is set to zero, but that has no // impact on routing decisions when the weight sum is zero tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); // One inactive deployment is put back in. Global DNS record now points to the only active deployment tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); // Setting zone (containing active deployment) out puts all deployments in tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.out); context.flushDnsUpdates(); assertEquals(RoutingStatus.Value.out, tester.routingPolicies().read(zone1).routingStatus().value()); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); // Setting zone back in removes the currently inactive deployment tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.in); context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); // Inactive deployment is set in tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); for (var policy : tester.routingPolicies().read(context.instanceId())) { assertSame(RoutingStatus.Value.in, policy.status().routingStatus().value()); } tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); } @Test public void application_endpoint_routing_policy() { RoutingPoliciesTester tester = new RoutingPoliciesTester(); TenantAndApplicationId application = TenantAndApplicationId.from("tenant1", "app1"); ApplicationId betaInstance = application.instance("beta"); ApplicationId mainInstance = application.instance("main"); DeploymentContext betaContext = tester.newDeploymentContext(betaInstance); DeploymentContext mainContext = tester.newDeploymentContext(mainInstance); var applicationPackage = applicationPackageBuilder() .instances("beta,main") .region(zone1.region()) .region(zone2.region()) .applicationEndpoint("a0", "c0", "us-west-1", Map.of(betaInstance.instance(), 2, mainInstance.instance(), 8)) .applicationEndpoint("a1", "c1", "us-central-1", Map.of(betaInstance.instance(), 4, mainInstance.instance(), 6)) .build(); for (var zone : List.of(zone1, zone2)) { tester.provisionLoadBalancers(2, betaInstance, zone); tester.provisionLoadBalancers(2, mainInstance, zone); } // Deploy both instances betaContext.submit(applicationPackage).deploy(); // Application endpoint points to both instances with correct weights DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1); DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1); DeploymentId betaZone2 = betaContext.deploymentIdIn(zone2); DeploymentId mainZone2 = mainContext.deploymentIdIn(zone2); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 2, mainZone1, 8)); tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1, Map.of(betaZone2, 4, mainZone2, 6)); // Weights are updated applicationPackage = applicationPackageBuilder() .instances("beta,main") .region(zone1.region()) .region(zone2.region()) .applicationEndpoint("a0", "c0", "us-west-1", Map.of(betaInstance.instance(), 3, mainInstance.instance(), 7)) .applicationEndpoint("a1", "c1", "us-central-1", Map.of(betaInstance.instance(), 1, mainInstance.instance(), 9)) .build(); betaContext.submit(applicationPackage).deploy(); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 3, mainZone1, 7)); tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1, Map.of(betaZone2, 1, mainZone2, 9)); // An endpoint is removed applicationPackage = applicationPackageBuilder() .instances("beta,main") .region(zone1.region()) .region(zone2.region()) .applicationEndpoint("a0", "c0", "us-west-1", Map.of(betaInstance.instance(), 1)) .build(); betaContext.submit(applicationPackage).deploy(); // Application endpoints now point to a single instance tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 1)); assertTrue("Endpoint removed", tester.controllerTester().controller().routing() .readDeclaredEndpointsOf(application) .named(EndpointId.of("a1")).isEmpty()); } @Test public void application_endpoint_routing_status() { RoutingPoliciesTester tester = new RoutingPoliciesTester(); TenantAndApplicationId application = TenantAndApplicationId.from("tenant1", "app1"); ApplicationId betaInstance = application.instance("beta"); ApplicationId mainInstance = application.instance("main"); DeploymentContext betaContext = tester.newDeploymentContext(betaInstance); DeploymentContext mainContext = tester.newDeploymentContext(mainInstance); var applicationPackage = applicationPackageBuilder() .instances("beta,main") .region(zone1.region()) .applicationEndpoint("a0", "c0", "us-west-1", Map.of(betaInstance.instance(), 2, mainInstance.instance(), 8)) .build(); tester.provisionLoadBalancers(1, betaInstance, zone1); tester.provisionLoadBalancers(1, mainInstance, zone1); // Deploy both instances betaContext.submit(applicationPackage).deploy(); // Application endpoint points to both instances with correct weights DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1); DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 2, mainZone1, 8)); // Changing routing status removes deployment from DNS tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); betaContext.flushDnsUpdates(); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 2)); // Changing routing status for remaining deployment adds back all deployments, because removing all deployments // puts all IN tester.routingPolicies().setRoutingStatus(betaZone1, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); betaContext.flushDnsUpdates(); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 2, mainZone1, 8)); // Activating main deployment allows us to deactivate the beta deployment tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.in, RoutingStatus.Agent.tenant); betaContext.flushDnsUpdates(); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(mainZone1, 8)); // Activate all deployments again tester.routingPolicies().setRoutingStatus(betaZone1, RoutingStatus.Value.in, RoutingStatus.Agent.tenant); betaContext.flushDnsUpdates(); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(betaZone1, 2, mainZone1, 8)); } /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ private static ApplicationPackageBuilder applicationPackageBuilder() { return new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")); } private static List createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { List loadBalancers = new ArrayList<>(); for (int i = 0; i < count; i++) { HostName lbHostname; if (shared) { lbHostname = HostName.of("shared-lb--" + zone.value()); } else { lbHostname = HostName.of("lb-" + i + "--" + application.toFullString() + "--" + zone.value()); } loadBalancers.add( new LoadBalancer("LB-" + i + "-Z-" + zone.value(), application, ClusterSpec.Id.from("c" + i), Optional.of(lbHostname), LoadBalancer.State.active, Optional.of("dns-zone-1"))); } return loadBalancers; } private static List publicZones() { var sharedRegion = RegionName.from("aws-us-east-1c"); return List.of(ZoneId.from(Environment.prod, sharedRegion), ZoneId.from(Environment.prod, RegionName.from("aws-eu-west-1a")), ZoneId.from(Environment.staging, RegionName.from("us-east-3")), ZoneId.from(Environment.test, RegionName.from("us-east-1"))); } private static class RoutingPoliciesTester { private final DeploymentTester tester; public RoutingPoliciesTester() { this(SystemName.main); } public RoutingPoliciesTester(SystemName system) { this(new DeploymentTester(system.isPublic() ? new ControllerTester(new RotationsConfig.Builder().build(), system) : new ControllerTester(system)), true); } public RoutingPoliciesTester(DeploymentTester tester, boolean exclusiveRouting) { this.tester = tester; List zones; if (tester.controller().system().isPublic()) { zones = publicZones(); } else { zones = new ArrayList<>(tester.controllerTester().zoneRegistry().zones().all().ids()); // Default zones zones.add(zone4); // Missing from default ZoneRegistryMock zones } tester.controllerTester().setZones(zones); if (exclusiveRouting) { tester.controllerTester().setRoutingMethod(zones, RoutingMethod.exclusive); } } public RoutingPolicies routingPolicies() { return tester.controllerTester().controller().routing().policies(); } public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { return tester.newDeploymentContext(tenant, application, instance); } public DeploymentContext newDeploymentContext(ApplicationId instance) { return tester.newDeploymentContext(instance); } public ControllerTester controllerTester() { return tester.controllerTester(); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { for (ZoneId zone : zones) { tester.configServer().removeLoadBalancers(application, zone); tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, shared, clustersPerZone)); } } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { provisionLoadBalancers(clustersPerZone, application, false, zones); } private RoutingPolicyList policiesOf(ApplicationId instance) { return tester.controller().routing().policies().read(instance); } private Set recordNames() { return tester.controllerTester().nameService().records().stream() .map(Record::name) .map(RecordName::asString) .collect(Collectors.toSet()); } private Set aliasDataOf(String name) { return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream() .map(Record::data) .map(RecordData::asString) .collect(Collectors.toSet()); } private List cnameDataOf(String name) { return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream() .map(Record::data) .map(RecordData::asString) .collect(Collectors.toList()); } /** Assert that an application endpoint points to given targets and weights */ private void assertTargets(TenantAndApplicationId application, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map deploymentWeights) { Map> deploymentsByDnsName = new HashMap<>(); for (var deployment : deploymentWeights.keySet()) { EndpointList applicationEndpoints = tester.controller().routing().readDeclaredEndpointsOf(application) .named(endpointId) .targets(deployment) .cluster(cluster); assertEquals("Expected a single endpoint with ID '" + endpointId + "'", 1, applicationEndpoints.size()); String dnsName = applicationEndpoints.asList().get(0).dnsName(); deploymentsByDnsName.computeIfAbsent(dnsName, (k) -> new ArrayList<>()) .add(deployment); } assertEquals("Found " + endpointId + " for " + application, 1, deploymentsByDnsName.size()); deploymentsByDnsName.forEach((dnsName, deployments) -> { Set weightedTargets = deployments.stream() .map(d -> "weighted/lb-" + loadBalancerId + "--" + d.applicationId().toFullString() + "--" + d.zoneId().value() + "/dns-zone-1/" + d.zoneId().value() + "/" + deploymentWeights.get(d)) .collect(Collectors.toSet()); assertEquals(dnsName + " has expected targets", weightedTargets, aliasDataOf(dnsName)); }); } /** Assert that an instance endpoint points to given targets and weights */ private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map zoneWeights) { Set latencyTargets = new HashSet<>(); Map> zonesByRegionEndpoint = new HashMap<>(); for (var zone : zoneWeights.keySet()) { DeploymentId deployment = new DeploymentId(instance, zone); EndpointList regionEndpoints = tester.controller().routing().readEndpointsOf(deployment) .cluster(cluster) .scope(Endpoint.Scope.weighted); Endpoint regionEndpoint = regionEndpoints.first().orElseThrow(() -> new IllegalArgumentException("No region endpoint found for " + cluster + " in " + deployment)); zonesByRegionEndpoint.computeIfAbsent(regionEndpoint.dnsName(), (k) -> new ArrayList<>()) .add(zone); } zonesByRegionEndpoint.forEach((regionEndpoint, zonesInRegion) -> { Set weightedTargets = zonesInRegion.stream() .map(z -> "weighted/lb-" + loadBalancerId + "--" + instance.toFullString() + "--" + z.value() + "/dns-zone-1/" + z.value() + "/" + zoneWeights.get(z)) .collect(Collectors.toSet()); assertEquals("Region endpoint " + regionEndpoint + " points to load balancer", weightedTargets, aliasDataOf(regionEndpoint)); ZoneId zone = zonesInRegion.get(0); String latencyTarget = "latency/" + regionEndpoint + "/dns-zone-1/" + zone.value(); latencyTargets.add(latencyTarget); }); List deployments = zoneWeights.keySet().stream().map(z -> new DeploymentId(instance, z)).collect(Collectors.toList()); String globalEndpoint = tester.controller().routing().readDeclaredEndpointsOf(instance) .named(endpointId) .targets(deployments) .primary() .map(Endpoint::dnsName) .orElse(""); assertEquals("Global endpoint " + globalEndpoint + " points to expected latency targets", latencyTargets, Set.copyOf(aliasDataOf(globalEndpoint))); } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId... zones) { Map zoneWeights = new LinkedHashMap<>(); for (var zone : zones) { zoneWeights.put(zone, 1L); } assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zoneWeights); } private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, Map zoneWeights) { assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zoneWeights); } } }