diff options
| author | Martin Polden <mpolden@mpolden.no> | 2020-07-01 11:14:37 +0200 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-07-01 11:14:37 +0200 |
| commit | b6e996367d997b601aec778359a59c8f206ff155 (patch) | |
| tree | 067adbd3a14bc50ee8b528dd1af37215270f8688 | |
| parent | 835faf3bd89204f5efb63b1133f27452fd1af242 (diff) | |
| parent | 4d2b583e8c9dd1cf5fa61ea7f0a6a199313b9962 (diff) | |
Merge pull request #13745 from vespa-engine/mpolden/weighted-alias
Support duplicate regions within same global endpoint
12 files changed, 1054 insertions, 130 deletions
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java index 7bd43ff1dcb..44acf1ab02e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java @@ -44,6 +44,11 @@ public class LatencyAliasTarget extends AliasTarget { return Objects.hash(super.hashCode(), zone); } + @Override + public String toString() { + return "latency target for " + name() + "[id=" + id() + ",dnsZone=" + dnsZone() + "]"; + } + /** Unpack latency alias from given record data */ public static LatencyAliasTarget unpack(RecordData data) { var parts = data.asString().split("/"); diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java index e8688b17347..03b10780e33 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java @@ -46,9 +46,10 @@ public class MemoryNameService implements NameService { .map(d -> new Record(Record.Type.ALIAS, name, d.pack())) .collect(Collectors.toList()); // Satisfy idempotency contract of interface - records.stream() - .filter(r -> !this.records.contains(r)) - .forEach(this::add); + for (var r1 : records) { + this.records.removeIf(r2 -> conflicts(r1, r2)); + } + this.records.addAll(records); return records; } @@ -115,10 +116,13 @@ public class MemoryNameService implements NameService { * most real name services. */ private static boolean conflicts(Record r1, Record r2) { - if (!r1.name().equals(r2.name())) return false; // Distinct names never conflict - if (r1.type() == Record.Type.ALIAS && r1.type() == r2.type()) // ALIAS records only require distinct data - return r1.data().equals(r2.data()); - return true; // Anything else is considered a conflict + if (!r1.name().equals(r2.name())) return false; // Distinct names never conflict + if (r1.type() == Record.Type.ALIAS && r1.type() == r2.type()) { + AliasTarget t1 = AliasTarget.unpack(r1.data()); + AliasTarget t2 = AliasTarget.unpack(r2.data()); + return t1.name().equals(t2.name()); // ALIAS records require distinct targets + } + return true; // Anything else is considered a conflict } } diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java index 9d741cb2dbc..8f81c94257e 100644 --- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java +++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java @@ -47,6 +47,11 @@ public class WeightedAliasTarget extends AliasTarget { return Objects.hash(super.hashCode(), weight); } + @Override + public String toString() { + return "weighted target for " + name() + "[id=" + id() + ",dnsZone=" + dnsZone() + ",weight=" + weight + "]"; + } + /** Unpack weighted alias from given record data */ public static WeightedAliasTarget unpack(RecordData data) { var parts = data.asString().split("/"); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 98169ba3196..3b1926fa6e9 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -82,7 +82,7 @@ public class RoutingController { return rotationRepository; } - /** Returns zone-scoped endpoints for given deployment */ + /** Returns endpoints for given deployment */ public EndpointList endpointsOf(DeploymentId deployment) { var endpoints = new LinkedHashSet<Endpoint>(); boolean isSystemApplication = SystemApplication.matching(deployment.applicationId()).isPresent(); @@ -93,6 +93,7 @@ public class RoutingController { for (var routingMethod : controller.zoneRegistry().routingMethods(policy.id().zone())) { if (routingMethod.isDirect() && !isSystemApplication && !canRouteDirectlyTo(deployment, application.get())) continue; endpoints.add(policy.endpointIn(controller.system(), routingMethod, controller.zoneRegistry())); + endpoints.add(policy.weightedEndpointIn(controller.system(), routingMethod)); } } return EndpointList.copyOf(endpoints); @@ -134,7 +135,7 @@ public class RoutingController { return EndpointList.copyOf(endpoints); } - /** Returns all non-global endpoints and corresponding cluster IDs for given deployments, grouped by their zone */ + /** Returns all zone-scoped endpoints and corresponding cluster IDs for given deployments, grouped by their zone */ public Map<ZoneId, List<Endpoint>> zoneEndpointsOf(Collection<DeploymentId> deployments) { var endpoints = new TreeMap<ZoneId, List<Endpoint>>(Comparator.comparing(ZoneId::value)); for (var deployment : deployments) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index f8982c96637..62804074337 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; @@ -36,15 +37,14 @@ public class Endpoint { private final RoutingMethod routingMethod; private final boolean tls; - private Endpoint(String name, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy, - RoutingMethod routingMethod) { + private Endpoint(String name, URI url, List<ZoneId> zones, Scope scope, Port port, boolean legacy, RoutingMethod routingMethod) { Objects.requireNonNull(name, "name must be non-null"); Objects.requireNonNull(zones, "zones must be non-null"); Objects.requireNonNull(scope, "scope must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); - if (scope == Scope.zone && zones.size() != 1) { - throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints"); + if ((scope == Scope.zone || scope == Scope.weighted) && zones.size() != 1) { + throw new IllegalArgumentException("A single zone must be given for " + scope + "-scoped endpoints"); } this.name = name; this.url = url; @@ -189,8 +189,10 @@ public class Endpoint { private static String scopePart(Scope scope, List<ZoneId> zones, boolean legacy) { if (scope == Scope.global) return "global"; var zone = zones.get(0); - if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints - return zone.region().value() + "." + zone.environment().value(); + var region = zone.region().value(); + if (scope == Scope.weighted) region += "-w"; + if (!legacy && zone.environment().isProduction()) return region; // Skip prod environment for non-legacy endpoints + return region + "." + zone.environment().value(); } private static String instancePart(ApplicationId application, String separator) { @@ -241,6 +243,21 @@ public class Endpoint { return part.substring(Math.max(0, part.length() - 63)); } + /** Returns the given region without availability zone */ + private static RegionName effectiveRegion(RegionName region) { + if (region.value().isEmpty()) return region; + String value = region.value(); + char lastChar = value.charAt(value.length() - 1); + if (lastChar >= 'a' && lastChar <= 'z') { // Remove availability zone + value = value.substring(0, value.length() - 1); + } + return RegionName.from(value); + } + + private static ZoneId effectiveZone(ZoneId zone) { + return ZoneId.from(zone.environment(), effectiveRegion(zone.region())); + } + /** An endpoint's scope */ public enum Scope { @@ -250,6 +267,9 @@ public class Endpoint { /** Endpoint points to a single zone */ zone, + /** Endpoint points to a single region */ + weighted, + } /** Represents an endpoint's HTTP port */ @@ -388,6 +408,16 @@ public class Endpoint { return this; } + /** Make this a weighted endpoint */ + public EndpointBuilder weighted() { + if (scope != Scope.zone && scope != Scope.weighted) { + throw new IllegalArgumentException("Endpoint must target zone before making it weighted"); + } + this.scope = Scope.weighted; + this.zones = List.of(effectiveZone(zones.get(0))); + return this; + } + /** Sets the system that owns this */ public Endpoint in(SystemName system) { String name; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 54004685cf1..dc3c14c76b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -44,11 +44,11 @@ import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbi import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname; import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Cluster; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.Application; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; @@ -95,7 +95,6 @@ import javax.ws.rs.NotAuthorizedException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.math.RoundingMode; import java.net.URI; import java.net.URISyntaxException; import java.security.DigestInputStream; @@ -104,8 +103,6 @@ import java.security.PublicKey; import java.time.DayOfWeek; import java.time.Duration; import java.time.Instant; -import java.time.YearMonth; -import java.time.format.DateTimeParseException; import java.util.Arrays; import java.util.Base64; import java.util.Comparator; @@ -116,7 +113,6 @@ import java.util.Map; import java.util.Optional; import java.util.OptionalLong; import java.util.Scanner; -import java.util.Set; import java.util.StringJoiner; import java.util.logging.Level; import java.util.stream.Collectors; @@ -975,7 +971,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add zone endpoints var endpointArray = response.setArray("endpoints"); - for (var endpoint : controller.routing().endpointsOf(deploymentId)) { + for (var endpoint : controller.routing().endpointsOf(deploymentId).scope(Endpoint.Scope.zone)) { toSlime(endpoint, endpoint.name(), endpointArray.addObject()); } // Add global endpoints diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index e080b7babce..2fa931f2219 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -3,9 +3,13 @@ package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; @@ -14,6 +18,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; @@ -42,10 +48,12 @@ public class RoutingPolicies { private final Controller controller; private final CuratorDb db; + private final BooleanFlag weightedDnsPerRegion; public RoutingPolicies(Controller controller) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.db = controller.curator(); + this.weightedDnsPerRegion = Flags.WEIGHTED_DNS_PER_REGION.bindTo(controller.flagSource()); try (var lock = db.lockRoutingPolicies()) { // Update serialized format for (var policy : db.readRoutingPolicies().entrySet()) { db.writeRoutingPolicies(policy.getKey(), policy.getValue()); @@ -84,7 +92,7 @@ public class RoutingPolicies { removeGlobalDnsUnreferencedBy(allocation, lock); storePoliciesOf(allocation, lock); removePoliciesUnreferencedBy(allocation, lock); - updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock); + updateGlobalDnsOf(application, get(allocation.deployment.applicationId()).values(), inactiveZones, lock); } } @@ -93,9 +101,9 @@ public class RoutingPolicies { try (var lock = db.lockRoutingPolicies()) { db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, controller.clock().instant()))); - var allPolicies = db.readRoutingPolicies(); - for (var applicationPolicies : allPolicies.values()) { - updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock); + Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> allPolicies = db.readRoutingPolicies(); + for (var kv : allPolicies.entrySet()) { + updateGlobalDnsOf(kv.getKey(), kv.getValue().values(), Set.of(), lock); } } } @@ -112,12 +120,12 @@ public class RoutingPolicies { newPolicies.put(policy.id(), newPolicy); } db.writeRoutingPolicies(deployment.applicationId(), newPolicies); - updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); + updateGlobalDnsOf(deployment.applicationId(), newPolicies.values(), Set.of(), lock); } } /** Update global DNS record for given policies */ - private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + private void legacyUpdateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { // Create DNS record for each routing ID var routingTable = routingTableFrom(routingPolicies); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { @@ -156,6 +164,66 @@ public class RoutingPolicies { } } + // TODO(mpolden): Remove and inline call to updateGlobalDnsOf when feature flag disappears + private void updateGlobalDnsOf(ApplicationId application, Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + if (weightedDnsPerRegion.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value()) { + updateGlobalDnsOf(routingPolicies, inactiveZones, lock); + } else { + legacyUpdateGlobalDnsOf(routingPolicies, inactiveZones, lock); + } + } + + /** Update global DNS record for given policies */ + private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(routingPolicies); + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + Map<RegionEndpoint, Set<AliasTarget>> targets = computeRegionEndpoints(routeEntry.getValue(), inactiveZones); + // Create a weighted ALIAS per region, pointing to all zones within the same region + targets.forEach(((regionEndpoint, weightedTargets) -> { + controller.nameServiceForwarder().createAlias(RecordName.from(regionEndpoint.dnsName), weightedTargets, + Priority.normal); + })); + // Create global latency-based ALIAS pointing to each per-region weighted ALIAS + var endpoints = controller.routing().endpointsOf(routeEntry.getKey().application()) + .named(routeEntry.getKey().endpointId()) + .not().requiresRotation(); + Set<AliasTarget> latencyTargets = targets.keySet().stream() + .map(regionEndpoint -> new LatencyAliasTarget(HostName.from(regionEndpoint.dnsName), + regionEndpoint.dnsZone, + regionEndpoint.zone)) + .collect(Collectors.toSet()); + endpoints.forEach(endpoint -> controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), + latencyTargets, Priority.normal)); + } + } + + /** Compute region endpoints and their targets from given policies */ + private Map<RegionEndpoint, Set<AliasTarget>> computeRegionEndpoints(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) { + Map<RegionEndpoint, Set<AliasTarget>> targets = new LinkedHashMap<>(); + RoutingMethod routingMethod = RoutingMethod.exclusive; + for (var policy : policies) { + if (policy.dnsZone().isEmpty()) continue; + if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(routingMethod)) continue; + Endpoint weighted = policy.weightedEndpointIn(controller.system(), routingMethod); + // Do not route to zone if global routing status is set out at: + // - zone level (ZoneRoutingPolicy) + // - deployment level (RoutingPolicy) + // - application package level (deployment.xml) + long weight = 1; + var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); + if (isConfiguredOut(policy, zonePolicy, inactiveZones)) { + weight = 0; // A record with 0 weight will not received traffic. If all records within a group have 0 + // weight, traffic is routed to all records with equal probability. + } + var regionEndpoint = new RegionEndpoint(weighted, policy.dnsZone().get(), policy.id().zone()); + var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(), + policy.id().zone(), weight); + targets.computeIfAbsent(regionEndpoint, (k) -> new LinkedHashSet<>()) + .add(weightedTarget); + } + return Collections.unmodifiableMap(targets); + } + /** Store routing policies for given load balancers */ private void storePoliciesOf(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) { var policies = new LinkedHashMap<>(get(allocation.deployment.applicationId())); @@ -267,6 +335,35 @@ public class RoutingPolicies { return false; } + /** Represents a region-wide endpoint */ + private static class RegionEndpoint { + + private final String dnsName; + private final String dnsZone; + private final ZoneId zone; + + public RegionEndpoint(Endpoint endpoint, String dnsZone, ZoneId zone) { + this.dnsName = Objects.requireNonNull(endpoint).dnsName(); + this.dnsZone = Objects.requireNonNull(dnsZone); + this.zone = Objects.requireNonNull(zone); + if (endpoint.scope() != Endpoint.Scope.weighted) throw new IllegalArgumentException("Region endpoint must be weighted"); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RegionEndpoint that = (RegionEndpoint) o; + return dnsName.equals(that.dnsName); + } + + @Override + public int hashCode() { + return Objects.hash(dnsName); + } + + } + /** Load balancers allocated to a deployment */ private static class LoadBalancerAllocation { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index c56a5f0bd66..7f4a707949b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -74,11 +74,12 @@ public class RoutingPolicy { Optional<Endpoint> infraEndpoint = SystemApplication.matching(id.owner()) .flatMap(app -> app.endpointIn(id.zone(), zoneRegistry)); if (infraEndpoint.isPresent()) return infraEndpoint.get(); - return Endpoint.of(id.owner()) - .target(id.cluster(), id.zone()) - .on(Port.fromRoutingMethod(routingMethod)) - .routingMethod(routingMethod) - .in(system); + return endpoint(routingMethod).in(system); + } + + /** Returns the weighted endpoint of this */ + public Endpoint weightedEndpointIn(SystemName system, RoutingMethod routingMethod) { + return endpoint(routingMethod).weighted().in(system); } @Override @@ -101,4 +102,11 @@ public class RoutingPolicy { id.zone().value()); } + private Endpoint.EndpointBuilder endpoint(RoutingMethod routingMethod) { + return Endpoint.of(id.owner()) + .target(id.cluster(), id.zone()) + .on(Port.fromRoutingMethod(routingMethod)) + .routingMethod(routingMethod); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index f968b8c76d7..2e57a5eaaa1 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -228,6 +228,44 @@ public class EndpointTest { } @Test + public void weighted_endpoints() { + var cluster = ClusterSpec.Id.from("default"); + Map<String, Endpoint> tests = Map.of( + "https://a1.t1.us-north-1-w.public.vespa.oath.cloud/", + Endpoint.of(app1) + .target(cluster, ZoneId.from("prod", "us-north-1a")) + .weighted() + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .in(SystemName.Public), + "https://a1.t1.us-north-2-w.public.vespa.oath.cloud/", + Endpoint.of(app1) + .target(cluster, ZoneId.from("prod", "us-north-2")) + .weighted() + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .in(SystemName.Public), + "https://a1.t1.us-north-2-w.test.public.vespa.oath.cloud/", + Endpoint.of(app1) + .target(cluster, ZoneId.from("test", "us-north-2")) + .weighted() + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.url().toString())); + Endpoint endpoint = Endpoint.of(app1) + .target(cluster, ZoneId.from("prod", "us-north-1a")) + .weighted() + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .in(SystemName.main); + assertEquals("Availability zone is removed from region", + "us-north-1", + endpoint.zones().get(0).region().value()); + } + + @Test public void upstream_name() { var zone = ZoneId.from("prod", "us-north-1"); var tests1 = Map.of( diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java new file mode 100644 index 00000000000..8479309c63b --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java @@ -0,0 +1,712 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.google.common.collect.Sets; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.application.api.ValidationId; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.AthenzDomain; +import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.Environment; +import com.yahoo.config.provision.HostName; +import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.RoutingController; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.SystemApplication; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock; +import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; +import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher; +import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.junit.Test; + +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author mortent + * @author mpolden + */ +// TODO(mpolden): Remove when weighted-dns-per-region flag is removed +public class RoutingPoliciesLegacyTest { + + private final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); + private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); + private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); + + private final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .build(); + + @Test + public void global_routing_policies() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); + int clustersPerZone = 2; + int numberOfDeployments = 2; + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0") + .endpoint("r1", "c0", "us-west-1") + .endpoint("r2", "c1") + .build(); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2); + + // Creates alias records + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); + assertEquals("Routing policy count is equal to cluster count", + numberOfDeployments * clustersPerZone, + tester.policiesOf(context1.instance().id()).size()); + + // Applications gains a new deployment + ApplicationPackage applicationPackage2 = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .region(zone3.region()) + .endpoint("r0", "c0") + .endpoint("r1", "c0", "us-west-1") + .endpoint("r2", "c1") + .build(); + numberOfDeployments++; + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); + context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Endpoints are updated to contain cluster in new deployment + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); + + // Another application is deployed with a single cluster and global endpoint + var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; + tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); + var applicationPackage3 = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0") + .build(); + context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // All endpoints for app1 are removed + ApplicationPackage applicationPackage4 = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .region(zone3.region()) + .allow(ValidationId.globalEndpointChange) + .build(); + context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); + var policies = tester.policiesOf(context1.instanceId()); + assertEquals(clustersPerZone * numberOfDeployments, policies.size()); + assertTrue("Rotation membership is removed from all policies", + policies.stream().allMatch(policy -> policy.endpoints().isEmpty())); + assertEquals("Rotations for " + context2.application() + " are not removed", 2, tester.aliasDataOf(endpoint4).size()); + } + + @Test + public void zone_routing_policies() { + zone_routing_policies(false); + zone_routing_policies(true); + } + + private void zone_routing_policies(boolean sharedRoutingLayer) { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant1", "app2", "default"); + + // Deploy application + int clustersPerZone = 2; + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Deployment creates records and policies for all clusters in all zones + Set<String> expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(4, tester.policiesOf(context1.instanceId()).size()); + + // Next deploy does nothing + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(4, tester.policiesOf(context1.instanceId()).size()); + + // Add 1 cluster in each zone and deploy + tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(6, tester.policiesOf(context1.instanceId()).size()); + + // Deploy another application + tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2); + context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList())); + assertEquals(4, tester.policiesOf(context2.instanceId()).size()); + + // Deploy removes cluster from app1 + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + + // Remove app2 completely + tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet() + .forEach(zone -> { + tester.controllerTester().configServer().removeLoadBalancers(context2.instanceId(), zone); + tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone); + }); + context2.flushDnsUpdates(); + expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty()); + assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size()); + } + + @Test + public void global_routing_policies_in_rotationless_system() { + var tester = new RoutingPoliciesTester(new DeploymentTester(new ControllerTester(new RotationsConfig.Builder().build()))); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + + var applicationPackage = applicationPackageBuilder() + .region(zone1.region().value()) + .endpoint("r0", "c0") + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; + assertEquals(endpoint + " points to c0 in all regions", + List.of("latency/lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), + tester.aliasDataOf(endpoint)); + assertTrue("No rotations assigned", context.application().instances().values().stream() + .map(Instance::rotations) + .allMatch(List::isEmpty)); + } + + @Test + public void manual_deployment_creates_routing_policy() { + // Empty application package is valid in manually deployed environments + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + var emptyApplicationPackage = new ApplicationPackageBuilder().build(); + var zone = ZoneId.from("dev", "us-east-1"); + var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); + tester.controllerTester().serviceRegistry().zoneRegistry() + .setZones(zoneApi) + .exclusiveRoutingIn(zoneApi); + + // Deploy to dev + tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none()); + assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec()); + context.flushDnsUpdates(); + + // Routing policy is created and DNS is updated + assertEquals(1, tester.policiesOf(context.instanceId()).size()); + assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + } + + @Test + public void manual_deployment_creates_routing_policy_with_non_empty_spec() { + // Initial deployment + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + context.submit(applicationPackage).deploy(); + var zone = ZoneId.from("dev", "us-east-1"); + var zoneApi = ZoneApiMock.from(zone.environment(), zone.region()); + tester.controllerTester().serviceRegistry().zoneRegistry() + .setZones(zoneApi) + .exclusiveRoutingIn(zoneApi); + var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); + assertEquals(prodRecords, tester.recordNames()); + + // Deploy to dev under different instance + var devInstance = context.application().id().instance("user"); + tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none()); + assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec()); + context.flushDnsUpdates(); + + // Routing policy is created and DNS is updated + assertEquals(1, tester.policiesOf(devInstance).size()); + assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); + } + + @Test + public void reprovisioning_load_balancer_preserves_cname_record() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Initial load balancer is provisioned + tester.provisionLoadBalancers(1, context.instanceId(), zone1); + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .build(); + + // Application is deployed + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + var expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(1, tester.policiesOf(context.instanceId()).size()); + + // Application is removed and the load balancer is deprovisioned + tester.controllerTester().controller().applications().deactivate(context.instanceId(), zone1); + tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1); + + // Load balancer for the same application is provisioned again, but with a different hostname + var newHostname = HostName.from("new-hostname"); + var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(), + context.instanceId(), + ClusterSpec.Id.from("c0"), + newHostname, + LoadBalancer.State.active, + Optional.of("dns-zone-1")); + tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer)); + + // Application redeployment preserves DNS record + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(1, tester.policiesOf(context.instanceId()).size()); + assertEquals("CNAME points to current load blancer", newHostname.value() + ".", + tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); + } + + @Test + public void set_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Global DNS record is created + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + // Global routing status is overridden in one zone + var changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + + // Inactive zone is removed from global DNS record + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Status details is stored in policy + var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Other zone remains in + var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent()); + assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); + + // Next deployment does not affect status + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Deployment is set back in + tester.controllerTester().clock().advance(Duration.ofHours(1)); + changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Deployment is set out through a new deployment.xml + var applicationPackage2 = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region(), false) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); + + // ... back in + var applicationPackage3 = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + } + + @Test + public void set_zone_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant2", "app2", "default"); + var contexts = List.of(context1, context2); + + // Deploy applications + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) + .build(); + for (var context : contexts) { + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } + + // Set zone out + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + for (var context : contexts) { + var policies = tester.routingPolicies().get(context.instanceId()); + assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, + policies.values().stream() + .map(RoutingPolicy::status) + .map(Status::globalRouting) + .map(GlobalRouting::status) + .allMatch(status -> status == GlobalRouting.Status.in)); + } + var changedAt = tester.controllerTester().clock().instant(); + var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); + assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status()); + assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt()); + + // Setting status per deployment does not affect status as entire zone is out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + + // Set single deployment out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + + // Set zone back in. Deployment set explicitly out, remains out, the rest are in + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } + + @Test + public void non_production_deployment_is_not_registered_in_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public); + + // Configure the system to use the same region for test, staging and prod + var sharedRegion = RegionName.from("aws-us-east-1c"); + var prodZone = ZoneId.from(Environment.prod, sharedRegion); + var stagingZone = ZoneId.from(Environment.staging, sharedRegion); + var testZone = ZoneId.from(Environment.test, sharedRegion); + var zones = List.of(ZoneApiMock.from(prodZone), + ZoneApiMock.from(stagingZone), + ZoneApiMock.from(testZone)); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .setRoutingMethod(zones, RoutingMethod.exclusive); + tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone), + SystemApplication.all()); + + var context = tester.tester.newDeploymentContext(); + var endpointId = EndpointId.of("r0"); + var applicationPackage = applicationPackageBuilder() + .trustDefaultCertificate() + .region(sharedRegion) + .endpoint(endpointId.id(), "default") + .build(); + + // Application starts deployment + context = context.submit(applicationPackage); + for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) { + context = context.runJob(testJob); + // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the + // deployment, we consume only one DNS update at a time here + do { + context = context.flushDnsUpdates(1); + tester.assertTargets(context.instanceId(), endpointId, 0); + } while (!tester.recordNames().isEmpty()); + } + + // Deployment completes + context.completeRollout(); + tester.assertTargets(context.instanceId(), endpointId, 0, prodZone); + } + + @Test + public void changing_global_routing_status_never_removes_all_members() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Global DNS record is created, pointing to all configured zones + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Global routing status is overridden for one deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + + // Setting other deployment out implicitly sets all deployments in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // One inactive deployment is put back in. Global DNS record now points to the only active deployment + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Setting zone (containing active deployment) out puts all deployments in + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); + context.flushDnsUpdates(); + assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // Setting zone back in removes the currently inactive deployment + tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + + // Inactive deployment is set in + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { + assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); + } + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + } + + @Test + public void config_server_routing_policy() { + var tester = new RoutingPoliciesTester(); + var app = SystemApplication.configServer.id(); + RecordName name = RecordName.from("cfg.prod.us-west-1.test.vip"); + tester.controllerTester().nameService().add(new Record(Record.Type.A, name, RecordData.from("192.0.2.1"))); + + tester.provisionLoadBalancers(1, app, zone1); + tester.routingPolicies().refresh(app, DeploymentSpec.empty, zone1); + new NameServiceDispatcher(tester.tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE).run(); + + List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.CNAME, name); + assertEquals(1, records.size()); + assertEquals(RecordData.from("lb-0--hosted-vespa:zone-config-servers:default--prod.us-west-1."), + records.get(0).data()); + } + + /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ + private static ApplicationPackageBuilder applicationPackageBuilder() { + return new ApplicationPackageBuilder() + .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) + .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION); + } + + private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) { + List<LoadBalancer> loadBalancers = new ArrayList<>(); + for (int i = 0; i < count; i++) { + HostName lbHostname; + if (shared) { + lbHostname = HostName.from("shared-lb--" + zone.value()); + } else { + lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() + + "--" + zone.value()); + } + loadBalancers.add( + new LoadBalancer("LB-" + i + "-Z-" + zone.value(), + application, + ClusterSpec.Id.from("c" + i), + lbHostname, + LoadBalancer.State.active, + Optional.of("dns-zone-1"))); + } + return loadBalancers; + } + + private static class RoutingPoliciesTester { + + private final DeploymentTester tester; + + public RoutingPoliciesTester() { + this(SystemName.main); + } + + public RoutingPoliciesTester(SystemName system) { + this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system)))); + } + + public RoutingPolicies routingPolicies() { + return tester.controllerTester().controller().routing().policies(); + } + + public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { + return tester.newDeploymentContext(tenant, application, instance); + } + + public ControllerTester controllerTester() { + return tester.controllerTester(); + } + + public RoutingPoliciesTester(DeploymentTester tester) { + this.tester = tester; + // Make all zones directly routed + tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); + } + + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { + for (ZoneId zone : zones) { + tester.configServer().removeLoadBalancers(application, zone); + tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, shared, clustersPerZone)); + } + } + + private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) { + provisionLoadBalancers(clustersPerZone, application, false, zones); + } + + private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { + return tester.controller().curator().readRoutingPolicies(instance).values(); + } + + private Set<String> recordNames() { + return tester.controllerTester().nameService().records().stream() + .map(Record::name) + .map(RecordName::asString) + .collect(Collectors.toSet()); + } + + private List<String> aliasDataOf(String name) { + return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream() + .map(Record::data) + .map(RecordData::asString) + .collect(Collectors.toList()); + } + + private List<String> cnameDataOf(String name) { + return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream() + .map(Record::data) + .map(RecordData::asString) + .collect(Collectors.toList()); + } + + private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { + var endpoint = tester.controller().routing().endpointsOf(application) + .named(endpointId) + .targets(List.of(zone)) + .primary() + .map(Endpoint::dnsName) + .orElse("<none>"); + var zoneTargets = Arrays.stream(zone) + .map(z -> "latency/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + + z.value() + "/dns-zone-1/" + z.value()) + .collect(Collectors.toSet()); + assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, + Set.copyOf(aliasDataOf(endpoint))); + } + + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 3b2fe95d0f7..ca4bcf1a354 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -13,16 +13,21 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; +import com.yahoo.config.provision.zone.ZoneApi; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; @@ -40,15 +45,16 @@ import java.time.Duration; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.stream.Collectors; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; import static org.junit.Assert.assertTrue; /** @@ -57,9 +63,10 @@ import static org.junit.Assert.assertTrue; */ public class RoutingPoliciesTest { - private final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); - private final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); - private final ZoneId zone3 = ZoneId.from("prod", "us-east-3"); + private static final ZoneId zone1 = ZoneId.from("prod", "us-west-1"); + private static final ZoneId zone2 = ZoneId.from("prod", "us-central-1"); + private static final ZoneId zone3 = ZoneId.from("prod", "aws-us-east-1a"); + private static final ZoneId zone4 = ZoneId.from("prod", "aws-us-east-1b"); private final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) @@ -138,6 +145,30 @@ public class RoutingPoliciesTest { } @Test + public void global_routing_policies_with_duplicate_region() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + int clustersPerZone = 2; + int numberOfDeployments = 3; + var applicationPackage = applicationPackageBuilder() + .region(zone1.region()) + .region(zone3.region()) + .region(zone4.region()) + .endpoint("r0", "c0") + .endpoint("r1", "c1") + .build(); + tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone3, zone4); + + // Creates alias records + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone3, zone4); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 1, zone1, zone3, zone4); + assertEquals("Routing policy count is equal to cluster count", + numberOfDeployments * clustersPerZone, + tester.policiesOf(context1.instance().id()).size()); + } + + @Test public void zone_routing_policies() { zone_routing_policies(false); zone_routing_policies(true); @@ -245,10 +276,7 @@ public class RoutingPoliciesTest { .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud"; - assertEquals(endpoint + " points to c0 in all regions", - List.of("latency/lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint)); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); assertTrue("No rotations assigned", context.application().instances().values().stream() .map(Instance::rotations) .allMatch(List::isEmpty)); @@ -367,9 +395,9 @@ public class RoutingPoliciesTest { GlobalRouting.Agent.tenant); context.flushDnsUpdates(); - // Inactive zone is removed from global DNS record - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + // Inactive zone is given zero weight + tester.assertWeight(0, context.instanceId(), 0, zone1); + tester.assertWeight(1, context.instanceId(), 0, zone2); // Status details is stored in policy var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); @@ -386,16 +414,15 @@ public class RoutingPoliciesTest { // Next deployment does not affect status context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + tester.assertWeight(0, context.instanceId(), 0, zone1); + tester.assertWeight(1, context.instanceId(), 0, zone2); // Deployment is set back in tester.controllerTester().clock().advance(Duration.ofHours(1)); changedAt = tester.controllerTester().clock().instant(); tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + tester.assertWeight(1, context.instanceId(), 0, zone1, zone2); policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); @@ -410,8 +437,8 @@ public class RoutingPoliciesTest { .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertWeight(1, context.instanceId(), 0, zone1); + tester.assertWeight(0, context.instanceId(), 0, zone2); // ... back in var applicationPackage3 = applicationPackageBuilder() @@ -421,8 +448,7 @@ public class RoutingPoliciesTest { .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) .build(); context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + tester.assertWeight(1, context.instanceId(), 0, zone1, zone2); } @Test @@ -442,13 +468,14 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + tester.assertWeight(1, context.instanceId(), 0, zone1, zone2); } // Set zone out tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); context1.flushDnsUpdates(); - tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); - tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertWeight(1, context1.instanceId(), 0, zone1); + tester.assertWeight(0, context1.instanceId(), 0, zone2); for (var context : contexts) { var policies = tester.routingPolicies().get(context.instanceId()); assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, @@ -467,8 +494,8 @@ public class RoutingPoliciesTest { // Setting status per deployment does not affect status as entire zone is out tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); context1.flushDnsUpdates(); - tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); - tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertWeight(0, context1.instanceId(), 0, zone2); + tester.assertWeight(0, context2.instanceId(), 0, zone2); // Set single deployment out tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); @@ -477,8 +504,9 @@ public class RoutingPoliciesTest { // Set zone back in. Deployment set explicitly out, remains out, the rest are in tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); context1.flushDnsUpdates(); - tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); - tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + tester.assertWeight(1, context1.instanceId(), 0, zone1); + tester.assertWeight(0, context1.instanceId(), 0, zone2); + tester.assertWeight(1, context2.instanceId(), 0, zone1, zone2); } @Test @@ -521,63 +549,7 @@ public class RoutingPoliciesTest { // Deployment completes context.completeRollout(); - tester.assertTargets(context.instanceId(), endpointId, 0, prodZone); - } - - @Test - public void changing_global_routing_status_never_removes_all_members() { - var tester = new RoutingPoliciesTester(); - var context = tester.newDeploymentContext("tenant1", "app1", "default"); - - // Provision load balancers and deploy application - tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); - var applicationPackage = applicationPackageBuilder() - .region(zone1.region()) - .region(zone2.region()) - .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) - .build(); - context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - - // Global DNS record is created, pointing to all configured zones - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // Global routing status is overridden for one deployment - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - - // Setting other deployment out implicitly sets all deployments in - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // One inactive deployment is put back in. Global DNS record now points to the only active deployment - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - - // Setting zone (containing active deployment) out puts all deployments in - tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out); - context.flushDnsUpdates(); - assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status()); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); - - // Setting zone back in removes the currently inactive deployment - tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in); - context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - - // Inactive deployment is set in - tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in, - GlobalRouting.Agent.tenant); - context.flushDnsUpdates(); - for (var policy : tester.routingPolicies().get(context.instanceId()).values()) { - assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status()); - } - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), endpointId, ClusterSpec.Id.from("default"), 0, prodZone); } @Test @@ -651,8 +623,15 @@ public class RoutingPoliciesTest { public RoutingPoliciesTester(DeploymentTester tester) { this.tester = tester; - // Make all zones directly routed - tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones()); + List<ZoneApi> zones = new ArrayList<>(tester.controllerTester().zoneRegistry().zones().all().zones()); + zones.add(ZoneApiMock.from(zone3)); + zones.add(ZoneApiMock.from(zone4)); + tester.controllerTester().zoneRegistry() + .setZones(zones) + .exclusiveRoutingIn(zones); + tester.controllerTester().configServer().bootstrap(tester.controllerTester().zoneRegistry().zones().all().ids(), + SystemApplication.all()); + ((InMemoryFlagSource) tester.controllerTester().controller().flagSource()).withBooleanFlag(Flags.WEIGHTED_DNS_PER_REGION.id(), true); } private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) { @@ -691,19 +670,61 @@ public class RoutingPoliciesTest { .collect(Collectors.toList()); } - private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { - var endpoint = tester.controller().routing().endpointsOf(application) - .named(endpointId) - .targets(List.of(zone)) - .primary() - .map(Endpoint::dnsName) - .orElse("<none>"); - var zoneTargets = Arrays.stream(zone) - .map(z -> "latency/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + - z.value() + "/dns-zone-1/" + z.value()) - .collect(Collectors.toSet()); - assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, - Set.copyOf(aliasDataOf(endpoint))); + private void assertWeight(long expected, ApplicationId application, int loadBalancerId, ZoneId... zones) { + for (var zone : zones) { + Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone)) + .scope(Endpoint.Scope.weighted) + .named(EndpointId.of("c" + loadBalancerId)) + .asList() + .get(0); + List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, + RecordName.from(weighted.dnsName())); + assertEquals(1, records.size()); + assertEquals("Record " + weighted.dnsName() + " has expected weight", + expected, + WeightedAliasTarget.unpack(records.get(0).data()) + .weight()); + } + } + + private void assertTargets(ApplicationId application, EndpointId endpointId, ClusterSpec.Id clusterId, int loadBalancerId, ZoneId... zones) { + Set<String> latencyTargets = new HashSet<>(); + Map<String, List<ZoneId>> zonesByRegionEndpoint = new HashMap<>(); + for (var zone : zones) { + Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone)) + .scope(Endpoint.Scope.weighted) + .named(EndpointId.of(clusterId.value())) + .asList() + .get(0); + zonesByRegionEndpoint.computeIfAbsent(weighted.dnsName(), (k) -> new ArrayList<>()) + .add(zone); + } + zonesByRegionEndpoint.forEach((regionEndpoint, zonesInRegion) -> { + List<String> weightedTargets = zonesInRegion.stream() + .map(z -> "weighted/lb-" + loadBalancerId + "--" + + application.serializedForm() + "--" + z.value() + + "/dns-zone-1/" + z.value() + "/1") + .collect(Collectors.toList()); + assertEquals("Weighted endpoint " + regionEndpoint + " points to load balancer", + weightedTargets, + aliasDataOf(regionEndpoint)); + ZoneId zone = zonesInRegion.get(0); + String latencyTarget = "latency/" + regionEndpoint + "/dns-zone-1/" + zone.value(); + latencyTargets.add(latencyTarget); + }); + String globalEndpoint = tester.controller().routing().endpointsOf(application) + .named(endpointId) + .targets(List.of(zones)) + .primary() + .map(Endpoint::dnsName) + .orElse("<none>"); + assertEquals("Global endpoint " + globalEndpoint + " points to expected latency targets", + latencyTargets, Set.copyOf(aliasDataOf(globalEndpoint))); + + } + + private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId... zones) { + assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zones); } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 64d020f4d46..9e27d0c4a0b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -295,6 +295,13 @@ public class Flags { "Takes effect on next tick" ); + public static final UnboundBooleanFlag WEIGHTED_DNS_PER_REGION = defineFeatureFlag( + "weighted-dns-per-region", false, + "Whether to create weighted DNS records per region in global endpoints", + "Takes effect on next deployment through controller", + APPLICATION_ID + ); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description, String modificationEffect, FetchVector.Dimension... dimensions) { |