diff options
author | Øyvind Grønnesby <oyving@verizonmedia.com> | 2020-01-21 15:29:05 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-01-21 15:29:05 +0100 |
commit | a8a1ec711376e8146a639cb860f8c4860f8f0215 (patch) | |
tree | 33cf6c1268cf06b98784fffa4772c343d1b7b6f9 | |
parent | df15e99f9d098ad7a6f1672be8c7736ea6bcdd53 (diff) | |
parent | c5bbc53a26674ed11ebc342e2f7470a7eb5342e6 (diff) |
Merge pull request #11859 from vespa-engine/mpolden/routing-control
Support overriding global routing status of routing policies
23 files changed, 997 insertions, 432 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index dfc9574fcd7..82120f13b75 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.google.common.collect.ImmutableList; @@ -59,7 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Versions; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.rotation.RotationLock; import com.yahoo.vespa.hosted.controller.rotation.RotationRepository; @@ -686,9 +686,9 @@ public class ApplicationController { catch (RuntimeException e) { log.log(Level.WARNING, "Failed to get endpoint information for " + id, e); } - return routingPolicies.get(id).stream() + return routingPolicies.get(id).values().stream() .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone) - .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(), + .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(), policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java index b23d16767be..c8cfc8ac286 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; @@ -531,7 +531,7 @@ public class JobController { DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system())); return controller.applications().getDeploymentEndpoints(testerId) .stream().findAny() - .or(() -> controller.applications().routingPolicies().get(testerId).stream() + .or(() -> controller.applications().routingPolicies().get(testerId).values().stream() .findAny() .map(policy -> policy.endpointIn(controller.system()).url())); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java deleted file mode 100644 index ee38b2c9516..00000000000 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java +++ /dev/null @@ -1,233 +0,0 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; - -import com.yahoo.config.application.api.DeploymentSpec; -import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.vespa.curator.Lock; -import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; -import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; -import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; -import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; -import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; -import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -/** - * Updates routing policies and their associated DNS records based on an deployment's load balancers. - * - * @author mortent - * @author mpolden - */ -public class RoutingPolicies { - - private static final Logger LOGGER = Logger.getLogger(RoutingPolicies.class.getName()); - - private final Controller controller; - private final CuratorDb db; - - public RoutingPolicies(Controller controller) { - this.controller = Objects.requireNonNull(controller, "controller must be non-null"); - this.db = controller.curator(); - try (var lock = db.lockRoutingPolicies()) { // Update serialized format - for (var policy : db.readRoutingPolicies().entrySet()) { - db.writeRoutingPolicies(policy.getKey(), policy.getValue()); - } - } - } - - /** Read all known routing policies for given instance */ - public Set<RoutingPolicy> get(ApplicationId application) { - return db.readRoutingPolicies(application); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(DeploymentId deployment) { - return get(deployment.applicationId(), deployment.zoneId()); - } - - /** Read all known routing policies for given deployment */ - public Set<RoutingPolicy> get(ApplicationId application, ZoneId zone) { - return db.readRoutingPolicies(application).stream() - .filter(policy -> policy.zone().equals(zone)) - .collect(Collectors.toUnmodifiableSet()); - } - - /** - * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if - * load balancers for given application have changed. - */ - public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { - if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; - var lbs = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer().getLoadBalancers(application, zone), - deploymentSpec); - try (var lock = db.lockRoutingPolicies()) { - removeObsoleteEndpointsFromDns(lbs, lock); - storePoliciesOf(lbs, lock); - removeObsoletePolicies(lbs, lock); - registerEndpointsInDns(lbs, lock); - } - } - - /** Create global endpoints for given route, if any */ - private void registerEndpointsInDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application)); - - // Create DNS record for each routing ID - for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { - Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().endpointId(), - controller.system()); - Set<AliasTarget> targets = routeEntry.getValue() - .stream() - .filter(policy -> policy.dnsZone().isPresent()) - .map(policy -> new AliasTarget(policy.canonicalName(), - policy.dnsZone().get(), - policy.zone())) - .collect(Collectors.toSet()); - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); - } - } - - /** Store routing policies for given route. Returns the persisted policies. */ - private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var policies = new LinkedHashSet<>(get(loadBalancers.application)); - for (LoadBalancer loadBalancer : loadBalancers.list) { - var endpointIds = loadBalancers.endpointIdsOf(loadBalancer); - var policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer, endpointIds); - if (!policies.add(policy)) { - // Update existing policy - policies.remove(policy); - policies.add(policy); - } - } - db.writeRoutingPolicies(loadBalancers.application, policies); - } - - /** Create a policy for given load balancer and register a CNAME for it */ - private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer, - Set<EndpointId> endpointIds) { - var routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(), - loadBalancer.dnsZone(), endpointIds, isActive(loadBalancer)); - var name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName()); - var data = RecordData.fqdn(loadBalancer.hostname().value()); - controller.nameServiceForwarder().createCname(name, data, Priority.normal); - return routingPolicy; - } - - /** Remove obsolete policies for given route and their CNAME records */ - private void removeObsoletePolicies(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var allPolicies = new LinkedHashSet<>(get(loadBalancers.application)); - var removalCandidates = new HashSet<>(allPolicies); - var activeLoadBalancers = loadBalancers.list.stream() - .map(LoadBalancer::hostname) - .collect(Collectors.toSet()); - // Remove active load balancers and irrelevant zones from candidates - removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) || - !policy.zone().equals(loadBalancers.zone)); - for (var policy : removalCandidates) { - var dnsName = policy.endpointIn(controller.system()).dnsName(); - controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); - allPolicies.remove(policy); - } - db.writeRoutingPolicies(loadBalancers.application, allPolicies); - } - - /** Remove unreferenced global endpoints for given route from DNS */ - private void removeObsoleteEndpointsFromDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { - var zonePolicies = get(loadBalancers.application, loadBalancers.zone); - var removalCandidates = routingTableFrom(zonePolicies).keySet(); - var activeRoutingIds = routingIdsFrom(loadBalancers); - removalCandidates.removeAll(activeRoutingIds); - for (var id : removalCandidates) { - var endpoint = RoutingPolicy.endpointOf(id.application(), id.endpointId(), controller.system()); - controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); - } - } - - /** Compute routing IDs from given load balancers */ - private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { - Set<RoutingId> routingIds = new LinkedHashSet<>(); - for (var loadBalancer : loadBalancers.list) { - for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { - routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); - } - } - return Collections.unmodifiableSet(routingIds); - } - - /** Compute a routing table from given policies */ - private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) { - var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); - for (var policy : routingPolicies) { - for (var rotation : policy.endpoints()) { - var id = new RoutingId(policy.owner(), rotation); - routingTable.putIfAbsent(id, new ArrayList<>()); - routingTable.get(id).add(policy); - } - } - return routingTable; - } - - private static boolean isActive(LoadBalancer loadBalancer) { - switch (loadBalancer.state()) { - case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early - // as possible - case active: return true; - } - return false; - } - - /** Load balancers allocated to a deployment */ - private static class AllocatedLoadBalancers { - - private final ApplicationId application; - private final ZoneId zone; - private final List<LoadBalancer> list; - private final DeploymentSpec deploymentSpec; - - private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, - DeploymentSpec deploymentSpec) { - this.application = application; - this.zone = zone; - this.list = List.copyOf(loadBalancers); - this.deploymentSpec = deploymentSpec; - } - - /** Compute all endpoint IDs for given load balancer */ - private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { - if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints - return Set.of(); - } - var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); - if (instanceSpec.isEmpty()) { - return Set.of(); - } - return instanceSpec.get().endpoints().stream() - .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) - .filter(endpoint -> endpoint.regions().contains(zone.region())) - .map(com.yahoo.config.application.api.Endpoint::endpointId) - .map(EndpointId::of) - .collect(Collectors.toSet()); - } - - } - -} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 22894a084b6..1a2ffc69249 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -1,4 +1,4 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.google.common.util.concurrent.UncheckedTimeoutException; @@ -18,19 +18,21 @@ import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.auditlog.AuditLog; import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.deployment.Step; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; import com.yahoo.vespa.hosted.controller.tenant.Tenant; import com.yahoo.vespa.hosted.controller.versions.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.controller.versions.VespaVersion; -import org.apache.zookeeper.data.Stat; import java.io.IOException; import java.io.UncheckedIOException; @@ -80,6 +82,7 @@ public class CuratorDb { private static final Path jobRoot = root.append("jobs"); private static final Path controllerRoot = root.append("controllers"); private static final Path routingPoliciesRoot = root.append("routingPolicies"); + private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies"); private static final Path applicationCertificateRoot = root.append("applicationCertificates"); private final StringSetSerializer stringSetSerializer = new StringSetSerializer(); @@ -93,6 +96,7 @@ public class CuratorDb { private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer(); private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer); private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer(); + private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer); private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer(); private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer(); @@ -485,19 +489,28 @@ public class CuratorDb { // -------------- Routing policies ---------------------------------------- - public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) { + public void writeRoutingPolicies(ApplicationId application, Map<RoutingPolicyId, RoutingPolicy> policies) { curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies))); } - public Map<ApplicationId, Set<RoutingPolicy>> readRoutingPolicies() { + public Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> readRoutingPolicies() { return curator.getChildren(routingPoliciesRoot).stream() .map(ApplicationId::fromSerializedForm) .collect(Collectors.toUnmodifiableMap(Function.identity(), this::readRoutingPolicies)); } - public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) { + public Map<RoutingPolicyId, RoutingPolicy> readRoutingPolicies(ApplicationId application) { return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime)) - .orElseGet(Collections::emptySet); + .orElseGet(Map::of); + } + + public void writeZoneRoutingPolicy(ZoneRoutingPolicy policy) { + curator.set(zoneRoutingPolicyPath(policy.zone()), asJson(zoneRoutingPolicySerializer.toSlime(policy))); + } + + public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) { + return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data)) + .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS)); } // -------------- Application web certificates ---------------------------- @@ -581,6 +594,8 @@ public class CuratorDb { return routingPoliciesRoot.append(application.serializedForm()); } + private static Path zoneRoutingPolicyPath(ZoneId zone) { return zoneRoutingPoliciesRoot.append(zone.value()); } + private static Path nameServiceQueuePath() { return root.append("nameServiceQueue"); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index 54a3ef7551a..2429c5ee8c5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; import com.yahoo.config.provision.ApplicationId; @@ -6,13 +6,20 @@ import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.slime.ArrayTraverser; +import com.yahoo.slime.Cursor; +import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; +import java.time.Instant; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; -import java.util.Set; +import java.util.Map; /** * Serializer and deserializer for a {@link RoutingPolicy}. @@ -35,45 +42,64 @@ public class RoutingPolicySerializer { private static final String zoneField = "zone"; private static final String dnsZoneField = "dnsZone"; private static final String rotationsField = "rotations"; - private static final String activeField = "active"; + private static final String loadBalancerActiveField = "active"; + private static final String globalRoutingField = "globalRouting"; + private static final String agentField = "agent"; + private static final String changedAtField = "changedAt"; + private static final String statusField = "status"; - public Slime toSlime(Set<RoutingPolicy> routingPolicies) { + public Slime toSlime(Map<RoutingPolicyId, RoutingPolicy> routingPolicies) { var slime = new Slime(); var root = slime.setObject(); var policyArray = root.setArray(routingPoliciesField); - routingPolicies.forEach(policy -> { + routingPolicies.values().forEach(policy -> { var policyObject = policyArray.addObject(); - policyObject.setString(clusterField, policy.cluster().value()); - policyObject.setString(zoneField, policy.zone().value()); + policyObject.setString(clusterField, policy.id().cluster().value()); + policyObject.setString(zoneField, policy.id().zone().value()); policyObject.setString(canonicalNameField, policy.canonicalName().value()); policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone)); var rotationArray = policyObject.setArray(rotationsField); policy.endpoints().forEach(endpointId -> { rotationArray.addString(endpointId.id()); }); - policyObject.setBool(activeField, policy.active()); + policyObject.setBool(loadBalancerActiveField, policy.status().isActive()); + globalRoutingToSlime(policy.status().globalRouting(), policyObject.setObject(globalRoutingField)); }); return slime; } - public Set<RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { - var policies = new LinkedHashSet<RoutingPolicy>(); + public Map<RoutingPolicyId, RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) { + var policies = new LinkedHashMap<RoutingPolicyId, RoutingPolicy>(); var root = slime.get(); var field = root.field(routingPoliciesField); field.traverse((ArrayTraverser) (i, inspect) -> { var endpointIds = new LinkedHashSet<EndpointId>(); inspect.field(rotationsField).traverse((ArrayTraverser) (j, endpointId) -> endpointIds.add(EndpointId.of(endpointId.asString()))); - var activeFieldInspector = inspect.field(activeField); - // TODO(mpolden): Remove field presence check after January 2020 - boolean active = !activeFieldInspector.valid() || activeFieldInspector.asBool(); - policies.add(new RoutingPolicy(owner, - ClusterSpec.Id.from(inspect.field(clusterField).asString()), - ZoneId.from(inspect.field(zoneField).asString()), - HostName.from(inspect.field(canonicalNameField).asString()), - Serializers.optionalString(inspect.field(dnsZoneField)), - endpointIds, active)); + var id = new RoutingPolicyId(owner, + ClusterSpec.Id.from(inspect.field(clusterField).asString()), + ZoneId.from(inspect.field(zoneField).asString())); + policies.put(id, new RoutingPolicy(id, + HostName.from(inspect.field(canonicalNameField).asString()), + Serializers.optionalString(inspect.field(dnsZoneField)), + endpointIds, + new Status(inspect.field(loadBalancerActiveField).asBool(), + globalRoutingFromSlime(inspect.field(globalRoutingField))))); }); - return Collections.unmodifiableSet(policies); + return Collections.unmodifiableMap(policies); + } + + public void globalRoutingToSlime(GlobalRouting globalRouting, Cursor object) { + object.setString(statusField, globalRouting.status().name()); + object.setString(agentField, globalRouting.agent().name()); + object.setLong(changedAtField, globalRouting.changedAt().toEpochMilli()); + } + + public GlobalRouting globalRoutingFromSlime(Inspector object) { + if (!object.valid()) return GlobalRouting.DEFAULT_STATUS; + var status = GlobalRouting.Status.valueOf(object.field(statusField).asString()); + var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString()); + var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH); + return new GlobalRouting(status, agent, changedAt); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java new file mode 100644 index 00000000000..6688d16ad14 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java @@ -0,0 +1,44 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.slime.Slime; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; + +import java.util.Objects; + +/** + * Serializer for {@link ZoneRoutingPolicy}. + * + * @author mpolden + */ +public class ZoneRoutingPolicySerializer { + + // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one + // (and rewrite all nodes on startup), changes to the serialized format must be made + // such that what is serialized on version N+1 can be read by version N: + // - ADDING FIELDS: Always ok + // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version. + // - CHANGING THE FORMAT OF A FIELD: Don't do it bro. + + private static final String GLOBAL_ROUTING_FIELD = "globalRouting"; + + private final RoutingPolicySerializer routingPolicySerializer; + + public ZoneRoutingPolicySerializer(RoutingPolicySerializer routingPolicySerializer) { + this.routingPolicySerializer = Objects.requireNonNull(routingPolicySerializer, "routingPolicySerializer must be non-null"); + } + + public ZoneRoutingPolicy fromSlime(ZoneId zone, Slime slime) { + var root = slime.get(); + return new ZoneRoutingPolicy(zone, routingPolicySerializer.globalRoutingFromSlime(root.field(GLOBAL_ROUTING_FIELD))); + } + + public Slime toSlime(ZoneRoutingPolicy policy) { + var slime = new Slime(); + var root = slime.setObject(); + routingPolicySerializer.globalRoutingToSlime(policy.globalRouting(), root.setObject(GLOBAL_ROUTING_FIELD)); + return slime; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 378013b5e6d..f6cf776cbfa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.Signatures; @@ -68,7 +68,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentCost; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus; @@ -804,9 +803,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .forEach(globalEndpointUrls::add); // Per-cluster endpoints. These are backed by load balancers. - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); + var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); for (var policy : routingPolicies) { - policy.rotationEndpointsIn(controller.system()).asList().stream() + policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalEndpointUrls::add); @@ -929,10 +928,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler { .ifPresent(rotation -> object.setString("rotationId", rotation.asString())); // Per-cluster rotations - Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id()); - for (RoutingPolicy policy : routingPolicies) { - if (!policy.active()) continue; - policy.rotationEndpointsIn(controller.system()).asList().stream() + var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values(); + for (var policy : routingPolicies) { + if (!policy.status().isActive()) continue; + policy.globalEndpointsIn(controller.system()).asList().stream() .map(Endpoint::url) .map(URI::toString) .forEach(globalRotationsArray::addString); @@ -1043,11 +1042,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler { // Add endpoint(s) defined by routing policies var endpointArray = response.setArray("endpoints"); - for (var policy : controller.applications().routingPolicies().get(deploymentId)) { - if (!policy.active()) continue; + for (var policy : controller.applications().routingPolicies().get(deploymentId).values()) { + if (!policy.status().isActive()) continue; Cursor endpointObject = endpointArray.addObject(); Endpoint endpoint = policy.endpointIn(controller.system()); - endpointObject.setString("cluster", policy.cluster().value()); + endpointObject.setString("cluster", policy.id().cluster().value()); endpointObject.setBool("tls", endpoint.tls()); endpointObject.setString("url", endpoint.url().toString()); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java new file mode 100644 index 00000000000..1b2cf4a7896 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java @@ -0,0 +1,85 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.time.Instant; +import java.util.Objects; + +/** + * Represents the global routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. This contains the + * time global routing status was last changed and who changed it. + * + * This is immutable. + * + * @author mpolden + */ +public class GlobalRouting { + + public static final GlobalRouting DEFAULT_STATUS = new GlobalRouting(Status.in, Agent.system, Instant.EPOCH); + + private final Status status; + private final Agent agent; + private final Instant changedAt; + + /** DO NOT USE. Public for serialization purposes */ + public GlobalRouting(Status status, Agent agent, Instant changedAt) { + this.status = Objects.requireNonNull(status, "status must be non-null"); + this.agent = Objects.requireNonNull(agent, "agent must be non-null"); + this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null"); + } + + /** The current status of this */ + public Status status() { + return status; + } + + /** The agent who last changed this */ + public Agent agent() { + return agent; + } + + /** The time this was last changed */ + public Instant changedAt() { + return changedAt; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + GlobalRouting that = (GlobalRouting) o; + return status == that.status && + agent == that.agent && + changedAt.equals(that.changedAt); + } + + @Override + public int hashCode() { + return Objects.hash(status, agent, changedAt); + } + + @Override + public String toString() { + return "status " + status + ", changed by " + agent + " @ " + changedAt; + } + + public static GlobalRouting status(Status status, Agent agent, Instant instant) { + return new GlobalRouting(status, agent, instant); + } + + // Used in serialization. Do not change. + public enum Status { + /** Status is determined by health checks **/ + in, + + /** Status is explicitly set to out */ + out, + } + + /** Agents that can change the state of global routing */ + public enum Agent { + operator, + tenant, + system, + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java index 7b0ec3d27ba..5543d0ea0b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java @@ -1,7 +1,8 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import java.util.Objects; @@ -42,4 +43,9 @@ public class RoutingId { return Objects.hash(application, endpointId); } + @Override + public String toString() { + return "routing id for " + endpointId + " of " + application; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java new file mode 100644 index 00000000000..c05152e7795 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -0,0 +1,288 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; +import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget; +import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; +import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Updates routing policies and their associated DNS records based on an deployment's load balancers. + * + * @author mortent + * @author mpolden + */ +public class RoutingPolicies { + + private final Controller controller; + private final CuratorDb db; + + public RoutingPolicies(Controller controller) { + this.controller = Objects.requireNonNull(controller, "controller must be non-null"); + this.db = controller.curator(); + try (var lock = db.lockRoutingPolicies()) { // Update serialized format + for (var policy : db.readRoutingPolicies().entrySet()) { + db.writeRoutingPolicies(policy.getKey(), policy.getValue()); + } + } + } + + /** Read all known routing policies for given instance */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application) { + return db.readRoutingPolicies(application); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) { + return get(deployment.applicationId(), deployment.zoneId()); + } + + /** Read all known routing policies for given deployment */ + public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) { + return db.readRoutingPolicies(application).entrySet() + .stream() + .filter(kv -> kv.getKey().zone().equals(zone)) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + /** + * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if + * load balancers for given application have changed. + */ + public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) { + if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return; + var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer() + .getLoadBalancers(application, zone), + deploymentSpec); + var inactiveZones = inactiveZones(application, deploymentSpec); + try (var lock = db.lockRoutingPolicies()) { + removeGlobalDnsUnreferencedBy(loadBalancers, lock); + storePoliciesOf(loadBalancers, lock); + removePoliciesUnreferencedBy(loadBalancers, lock); + updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock); + } + } + + /** Set the status of all global endpoints in given zone */ + public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) { + try (var lock = db.lockRoutingPolicies()) { + db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator, + controller.clock().instant()))); + var allPolicies = db.readRoutingPolicies(); + for (var applicationPolicies : allPolicies.values()) { + updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock); + } + } + } + + /** Set the status of all global endpoints for given deployment */ + public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) { + try (var lock = db.lockRoutingPolicies()) { + var policies = get(deployment.applicationId()); + var newPolicies = new LinkedHashMap<>(policies); + for (var policy : policies.values()) { + if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone + var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent, + controller.clock().instant()))); + newPolicies.put(policy.id(), newPolicy); + } + db.writeRoutingPolicies(deployment.applicationId(), newPolicies); + updateGlobalDnsOf(newPolicies.values(), Set.of(), lock); + } + } + + /** Update global DNS record for given policies */ + private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) { + // Create DNS record for each routing ID + var routingTable = routingTableFrom(routingPolicies); + for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { + var targets = new LinkedHashSet<AliasTarget>(); + var staleTargets = new LinkedHashSet<AliasTarget>(); + for (var policy : routeEntry.getValue()) { + if (policy.dnsZone().isEmpty()) continue; + var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone()); + var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); + // Remove target zone if global routing status is set out at: + // - zone level (ZoneRoutingPolicy) + // - deployment level (RoutingPolicy) + // - application package level (deployment.xml) + if (anyOut(zonePolicy.globalRouting(), policy.status().globalRouting()) || + inactiveZones.contains(policy.id().zone())) { + staleTargets.add(target); + } else { + targets.add(target); + } + } + if (!targets.isEmpty()) { + var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(), + routeEntry.getKey().endpointId(), controller.system()); + controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal); + } + staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal)); + } + } + + /** Store routing policies for given load balancers */ + private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = new LinkedHashMap<>(get(loadBalancers.application)); + for (LoadBalancer loadBalancer : loadBalancers.list) { + var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone); + var existingPolicy = policies.get(policyId); + var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(), + loadBalancers.endpointIdsOf(loadBalancer), + new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS)); + // Preserve global routing status for existing policy + if (existingPolicy != null) { + newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().globalRouting())); + } + updateZoneDnsOf(newPolicy); + policies.put(newPolicy.id(), newPolicy); + } + db.writeRoutingPolicies(loadBalancers.application, policies); + } + + /** Update zone DNS record for given policy */ + private void updateZoneDnsOf(RoutingPolicy policy) { + var name = RecordName.from(policy.endpointIn(controller.system()).dnsName()); + var data = RecordData.fqdn(policy.canonicalName().value()); + controller.nameServiceForwarder().createCname(name, data, Priority.normal); + } + + /** Remove policies and zone DNS records unreferenced by given load balancers */ + private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var policies = get(loadBalancers.application); + var newPolicies = new LinkedHashMap<>(policies); + var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet()); + for (var policy : policies.values()) { + // Leave active load balancers and irrelevant zones alone + if (activeLoadBalancers.contains(policy.canonicalName()) || + !policy.id().zone().equals(loadBalancers.zone)) continue; + + var dnsName = policy.endpointIn(controller.system()).dnsName(); + controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal); + newPolicies.remove(policy.id()); + } + db.writeRoutingPolicies(loadBalancers.application, newPolicies); + } + + /** Remove unreferenced global endpoints from DNS */ + private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) { + var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values(); + var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet()); + var activeRoutingIds = routingIdsFrom(loadBalancers); + removalCandidates.removeAll(activeRoutingIds); + for (var id : removalCandidates) { + var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system()); + controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal); + } + } + + /** Compute routing IDs from given load balancers */ + private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) { + Set<RoutingId> routingIds = new LinkedHashSet<>(); + for (var loadBalancer : loadBalancers.list) { + for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) { + routingIds.add(new RoutingId(loadBalancer.application(), endpointId)); + } + } + return Collections.unmodifiableSet(routingIds); + } + + /** Compute a routing table from given policies */ + private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) { + var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>(); + for (var policy : routingPolicies) { + for (var endpoint : policy.endpoints()) { + var id = new RoutingId(policy.id().owner(), endpoint); + routingTable.putIfAbsent(id, new ArrayList<>()); + routingTable.get(id).add(policy); + } + } + return Collections.unmodifiableMap(routingTable); + } + + private static boolean anyOut(GlobalRouting... globalRouting) { + return Arrays.stream(globalRouting) + .map(GlobalRouting::status) + .anyMatch(status -> status == GlobalRouting.Status.out); + } + + private static boolean isActive(LoadBalancer loadBalancer) { + switch (loadBalancer.state()) { + case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early + // as possible + case active: return true; + } + return false; + } + + /** Load balancers allocated to a deployment */ + private static class AllocatedLoadBalancers { + + private final ApplicationId application; + private final ZoneId zone; + private final List<LoadBalancer> list; + private final DeploymentSpec deploymentSpec; + + private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers, + DeploymentSpec deploymentSpec) { + this.application = application; + this.zone = zone; + this.list = List.copyOf(loadBalancers); + this.deploymentSpec = deploymentSpec; + } + + /** Compute all endpoint IDs for given load balancer */ + private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) { + if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints + return Set.of(); + } + var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance()); + if (instanceSpec.isEmpty()) { + return Set.of(); + } + return instanceSpec.get().endpoints().stream() + .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value())) + .filter(endpoint -> endpoint.regions().contains(zone.region())) + .map(com.yahoo.config.application.api.Endpoint::endpointId) + .map(EndpointId::of) + .collect(Collectors.toSet()); + } + + } + + /** Returns zones where global routing is declared inactive for instance through deploymentSpec */ + private static Set<ZoneId> inactiveZones(ApplicationId instance, DeploymentSpec deploymentSpec) { + var instanceSpec = deploymentSpec.instance(instance.instance()); + if (instanceSpec.isEmpty()) return Set.of(); + return instanceSpec.get().zones().stream() + .filter(zone -> zone.environment().isProduction()) + .filter(zone -> !zone.active()) + .map(zone -> ZoneId.from(zone.environment(), zone.region().get())) + .collect(Collectors.toUnmodifiableSet()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index 80a62d94f2e..b1b6d1ae58a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -1,60 +1,46 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.application; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; import java.util.Objects; import java.util.Optional; import java.util.Set; /** - * Represents the DNS routing policy for a load balancer. A routing policy is uniquely identified by its owner, cluster - * and zone. + * Represents the DNS routing policy for a {@link com.yahoo.vespa.hosted.controller.application.Deployment}. * * @author mortent * @author mpolden */ public class RoutingPolicy { - private final ApplicationId owner; - private final ClusterSpec.Id cluster; - private final ZoneId zone; + private final RoutingPolicyId id; private final HostName canonicalName; private final Optional<String> dnsZone; private final Set<EndpointId> endpoints; - private final boolean active; + private final Status status; /** DO NOT USE. Public for serialization purposes */ - public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName, - Optional<String> dnsZone, Set<EndpointId> endpoints, boolean active) { - this.owner = Objects.requireNonNull(owner, "owner must be non-null"); - this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); - this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, Set<EndpointId> endpoints, + Status status) { + this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null"); this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null")); - this.active = active; + this.status = Objects.requireNonNull(status, "status must be non-null"); } - /** The application owning this */ - public ApplicationId owner() { - return owner; - } - - /** The zone this applies to */ - public ZoneId zone() { - return zone; - } - - /** The cluster this applies to */ - public ClusterSpec.Id cluster() { - return cluster; + /** The ID of this */ + public RoutingPolicyId id() { + return id; } /** The canonical name for this (rhs of a CNAME or ALIAS record) */ @@ -72,19 +58,24 @@ public class RoutingPolicy { return endpoints; } - /** Returns whether this is active (the underlying load balancer is in an active state) */ - public boolean active() { - return active; + /** Returns the status of this */ + public Status status() { + return status; + } + + /** Returns a copy of this with status set to given status */ + public RoutingPolicy with(Status status) { + return new RoutingPolicy(id, canonicalName, dnsZone, endpoints, status); } /** Returns the endpoint of this */ public Endpoint endpointIn(SystemName system) { - return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system); + return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system); } - /** Returns rotation endpoints of this */ - public EndpointList rotationEndpointsIn(SystemName system) { - return EndpointList.of(endpoints.stream().map(endpointId -> endpointOf(owner, endpointId, system))); + /** Returns global endpoints which this is a member of */ + public EndpointList globalEndpointsIn(SystemName system) { + return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system))); } @Override @@ -92,25 +83,23 @@ public class RoutingPolicy { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; RoutingPolicy that = (RoutingPolicy) o; - return owner.equals(that.owner) && - cluster.equals(that.cluster) && - zone.equals(that.zone); + return id.equals(that.id); } @Override public int hashCode() { - return Objects.hash(owner, cluster, zone); + return Objects.hash(id); } @Override public String toString() { return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, endpoints, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(), - zone.value()); + dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(), + id.zone().value()); } - /** Returns the endpoint of given rotation */ - public static Endpoint endpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { + /** Creates a global endpoint for given application */ + public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) { return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java new file mode 100644 index 00000000000..06002e874f1 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java @@ -0,0 +1,57 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Unique identifier for a {@link RoutingPolicy}. + * + * @author mpolden + */ +public class RoutingPolicyId { + + private final ApplicationId owner; + private final ClusterSpec.Id cluster; + private final ZoneId zone; + + public RoutingPolicyId(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone) { + this.owner = Objects.requireNonNull(owner, "owner must be non-null"); + this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null"); + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + } + + /** The application owning this */ + public ApplicationId owner() { + return owner; + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The cluster this applies to */ + public ClusterSpec.Id cluster() { + return cluster; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + RoutingPolicyId that = (RoutingPolicyId) o; + return owner.equals(that.owner) && + cluster.equals(that.cluster) && + zone.equals(that.zone); + } + + @Override + public int hashCode() { + return Objects.hash(owner, cluster, zone); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java new file mode 100644 index 00000000000..51e59c7cf4f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java @@ -0,0 +1,53 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import java.util.Objects; + +/** + * Represents the status of a routing policy. + * + * This is immutable. + * + * @author mpolden + */ +public class Status { + + private final boolean active; + private final GlobalRouting globalRouting; + + /** DO NOT USE. Public for serialization purposes */ + public Status(boolean active, GlobalRouting globalRouting) { + this.active = active; + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** Returns whether this is considered active according to the load balancer status */ + public boolean isActive() { + return active; + } + + /** Return status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + /** Returns a copy of this with global routing changed */ + public Status with(GlobalRouting globalRouting) { + return new Status(active, globalRouting); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Status status = (Status) o; + return active == status.active && + globalRouting.equals(status.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(active, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java new file mode 100644 index 00000000000..262cacd325e --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java @@ -0,0 +1,49 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.zone.ZoneId; + +import java.util.Objects; + +/** + * Represents the DNS routing policy for a zone. This takes precedence over of an individual {@link RoutingPolicy}. + * + * This is immutable. + * + * @author mpolden + */ +public class ZoneRoutingPolicy { + + private final ZoneId zone; + private final GlobalRouting globalRouting; + + public ZoneRoutingPolicy(ZoneId zone, GlobalRouting globalRouting) { + this.zone = Objects.requireNonNull(zone, "zone must be non-null"); + this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null"); + } + + /** The zone this applies to */ + public ZoneId zone() { + return zone; + } + + /** The status of global routing */ + public GlobalRouting globalRouting() { + return globalRouting; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ZoneRoutingPolicy that = (ZoneRoutingPolicy) o; + return zone.equals(that.zone) && + globalRouting.equals(that.globalRouting); + } + + @Override + public int hashCode() { + return Objects.hash(zone, globalRouting); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java index f722eb4f6bb..7c3c30738d6 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java @@ -1,17 +1,13 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.versions; import com.yahoo.component.Version; -import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.application.ApplicationList; import com.yahoo.vespa.hosted.controller.application.InstanceList; -import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList; import java.time.Instant; import java.time.ZoneOffset; -import java.util.stream.Collectors; import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index dbeb96337d1..c83463bc1ea 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; @@ -356,7 +356,6 @@ public final class ControllerTester { return application; } - public void deploy(ApplicationId id, ZoneId zone) { deploy(id, zone, new ApplicationPackage(new byte[0])); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index e4b5e77b377..9b0706d184f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -101,13 +101,19 @@ public class ApplicationPackageBuilder { } public ApplicationPackageBuilder region(RegionName regionName) { - return region(regionName.value()); + return region(regionName, true); } public ApplicationPackageBuilder region(String regionName) { - environmentBody.append(" <region active='true'>"); - environmentBody.append(regionName); - environmentBody.append("</region>\n"); + return region(RegionName.from(regionName), true); + } + + public ApplicationPackageBuilder region(RegionName regionName, boolean active) { + environmentBody.append(" <region active='") + .append(active) + .append("'>") + .append(regionName.value()) + .append("</region>\n"); return this; } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java index 2d0b625dcb3..2792a59b523 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java @@ -1,10 +1,12 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.AthenzDomain; import com.yahoo.config.provision.AthenzService; +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.security.KeyAlgorithm; import com.yahoo.security.KeyUtils; @@ -26,9 +28,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.Deployment; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock; import com.yahoo.vespa.hosted.controller.maintenance.JobRunner; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; import javax.security.auth.x500.X500Principal; import java.math.BigInteger; @@ -38,6 +45,7 @@ import java.security.cert.X509Certificate; import java.time.Instant; import java.util.Collections; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -57,8 +65,8 @@ import static org.junit.Assert.assertTrue; * * References to this should be acquired through {@link DeploymentTester#newDeploymentContext}. * - * Tester code that is not specific to deployments should be added to either {@link ControllerTester} or - * {@link DeploymentTester} instead of this class. + * Tester code that is not specific to a single application's deployment context should be added to either + * {@link ControllerTester} or {@link DeploymentTester} instead of this class. * * @author mpolden * @author jonmv @@ -197,6 +205,28 @@ public class DeploymentContext { return this; } + /** Add a routing policy for this in given zone, with status set to active */ + public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) { + return addRoutingPolicy(instanceId, zone, active); + } + + /** Add a routing policy for tester instance of this in given zone, with status set to active */ + public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) { + return addRoutingPolicy(testerId.id(), zone, active); + } + + private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) { + var clusterId = "default" + (!active ? "-inactive" : ""); + var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone); + var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance)); + policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"), + Optional.empty(), + Set.of(EndpointId.of("c0")), + new Status(active, GlobalRouting.DEFAULT_STATUS))); + tester.controller().curator().writeRoutingPolicies(instance, policies); + return this; + } + /** Submit given application package for deployment */ public DeploymentContext submit(ApplicationPackage applicationPackage) { return submit(applicationPackage, defaultSourceRevision); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java index 51726035cb3..e052b967c31 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java @@ -1,11 +1,10 @@ -// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.deployment; import com.google.common.collect.ImmutableList; import com.yahoo.component.Version; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.AthenzDomain; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.NodeResources; import com.yahoo.config.provision.SystemName; @@ -24,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud; import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import org.junit.Before; import org.junit.Test; @@ -43,20 +41,18 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.Set; import java.util.concurrent.Executors; import java.util.concurrent.Future; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info; import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning; -import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.publicCdApplicationPackage; +import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded; import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished; -import static java.util.Collections.emptySet; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -208,19 +204,8 @@ public class InternalStepRunnerTest { tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system())); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); - - tester.controller().curator().writeRoutingPolicies(app.instanceId(), Set.of(new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("default"), - JobType.systemTest.zone(system()), - HostName.from("host"), - Optional.empty(), - emptySet(), true))); - tester.controller().curator().writeRoutingPolicies(app.testerId().id(), Set.of(new RoutingPolicy(app.testerId().id(), - ClusterSpec.Id.from("default"), - JobType.systemTest.zone(system()), - HostName.from("host"), - Optional.empty(), - emptySet(), true))); + app.addRoutingPolicy(JobType.systemTest.zone(system()), true); + app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true); tester.runner().run();; assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal)); assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index 23355bd6033..c9ec5adc98c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -1,22 +1,26 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.persistence; -import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableMap; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.config.SlimeUtils; import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; +import com.yahoo.vespa.hosted.controller.routing.Status; import org.junit.Test; +import java.time.Instant; import java.util.Iterator; +import java.util.Map; import java.util.Optional; import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; /** * @author mortent @@ -29,41 +33,46 @@ public class RoutingPolicySerializerTest { public void serialization() { var owner = ApplicationId.defaultId(); var endpoints = Set.of(EndpointId.of("r1"), EndpointId.of("r2")); - var policies = ImmutableSet.of(new RoutingPolicy(owner, - ClusterSpec.Id.from("my-cluster1"), - ZoneId.from("prod", "us-north-1"), + var id1 = new RoutingPolicyId(owner, + ClusterSpec.Id.from("my-cluster1"), + ZoneId.from("prod", "us-north-1")); + var id2 = new RoutingPolicyId(owner, + ClusterSpec.Id.from("my-cluster2"), + ZoneId.from("prod", "us-north-2")); + var policies = ImmutableMap.of(id1, new RoutingPolicy(id1, HostName.from("long-and-ugly-name"), Optional.of("zone1"), - endpoints, true), - new RoutingPolicy(owner, - ClusterSpec.Id.from("my-cluster2"), - ZoneId.from("prod", "us-north-2"), + endpoints, new Status(true, GlobalRouting.DEFAULT_STATUS)), + id2, new RoutingPolicy(id2, HostName.from("long-and-ugly-name-2"), Optional.empty(), - endpoints, false)); + endpoints, new Status(false, + new GlobalRouting(GlobalRouting.Status.out, + GlobalRouting.Agent.tenant, + Instant.ofEpochSecond(123))))); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); - for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext();) { + for (Iterator<RoutingPolicy> it1 = policies.values().iterator(), it2 = serialized.values().iterator(); it1.hasNext();) { var expected = it1.next(); var actual = it2.next(); - assertEquals(expected.owner(), actual.owner()); - assertEquals(expected.cluster(), actual.cluster()); - assertEquals(expected.zone(), actual.zone()); + assertEquals(expected.id(), actual.id()); assertEquals(expected.canonicalName(), actual.canonicalName()); assertEquals(expected.dnsZone(), actual.dnsZone()); assertEquals(expected.endpoints(), actual.endpoints()); - assertEquals(expected.active(), actual.active()); + assertEquals(expected.status(), actual.status()); } } + // TODO(mpolden): Remove after January 2020 @Test public void legacy_serialization() { - var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\"," + - "\"canonicalName\":\"lb-0\"," + - "\"dnsZone\":\"dns-zone-id\",\"rotations\":[]}]}"; - var serialized = serializer.fromSlime(ApplicationId.defaultId(), SlimeUtils.jsonToSlime(json)); - assertTrue(serialized.iterator().next().active()); - + var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\",\"canonicalName\":\"lb-host\",\"dnsZone\":\"dnsZoneId\",\"rotations\":[\"default\"],\"active\":true}]}"; + var owner = ApplicationId.defaultId(); + var serialized = serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json)); + var id = new RoutingPolicyId(owner, ClusterSpec.Id.from("default"), ZoneId.from("prod", "us-north-1")); + var expected = Map.of(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.of("dnsZoneId"), + Set.of(EndpointId.defaultId()), new Status(true, GlobalRouting.DEFAULT_STATUS))); + assertEquals(expected, serialized); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java new file mode 100644 index 00000000000..6a089c5e1b0 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java @@ -0,0 +1,29 @@ +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.persistence; + +import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.routing.GlobalRouting; +import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy; +import org.junit.Test; + +import java.time.Instant; + +import static org.junit.Assert.assertEquals; + +/** + * @author mpolden + */ +public class ZoneRoutingPolicySerializerTest { + + @Test + public void serialization() { + var serializer = new ZoneRoutingPolicySerializer(new RoutingPolicySerializer()); + var zone = ZoneId.from("prod", "us-north-1"); + var policy = new ZoneRoutingPolicy(zone, + GlobalRouting.status(GlobalRouting.Status.out, GlobalRouting.Agent.operator, + Instant.ofEpochMilli(123))); + var serialized = serializer.fromSlime(zone, serializer.toSlime(policy)); + assertEquals(policy, serialized); + } + +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java index 30be5d9b399..96681dc1c8b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java @@ -1,4 +1,4 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.hosted.controller.restapi.application; import ai.vespa.hosted.api.MultiPartStreamer; @@ -11,7 +11,6 @@ import com.yahoo.config.provision.ApplicationName; import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.Environment; -import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; @@ -52,8 +51,6 @@ import com.yahoo.vespa.hosted.controller.application.Change; import com.yahoo.vespa.hosted.controller.application.ClusterInfo; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; -import com.yahoo.vespa.hosted.controller.application.EndpointId; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; @@ -1434,18 +1431,8 @@ public class ApplicationApiTest extends ControllerContainerTest { .region("us-west-1") .build(); app.submit(applicationPackage).deploy(); - Set<RoutingPolicy> policies = Set.of(new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("default"), - ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - HostName.from("lb-0-canonical-name"), - Optional.of("dns-zone-1"), Set.of(EndpointId.of("c0")), true), - // Inactive policy is not included - new RoutingPolicy(app.instanceId(), - ClusterSpec.Id.from("deleted-cluster"), - ZoneId.from(Environment.prod, RegionName.from("us-west-1")), - HostName.from("lb-1-canonical-name"), - Optional.of("dns-zone-1"), Set.of(), false)); - tester.controller().curator().writeRoutingPolicies(app.instanceId(), policies); + app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true); + app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false); // GET application tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 1bb20296bd2..c0420c7b895 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1,5 +1,5 @@ -// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -package com.yahoo.vespa.hosted.controller.maintenance; +// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; @@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; import com.yahoo.vespa.hosted.controller.application.ApplicationPackage; -import com.yahoo.vespa.hosted.controller.application.RoutingPolicy; +import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -24,7 +24,12 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.Test; import java.net.URI; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Map; @@ -68,21 +73,9 @@ public class RoutingPoliciesTest { // Creates alias records context1.submit(applicationPackage).deploy(); - var endpoint1 = "r0.app1.tenant1.global.vespa.oath.cloud"; - var endpoint2 = "r1.app1.tenant1.global.vespa.oath.cloud"; - var endpoint3 = "r2.app1.tenant1.global.vespa.oath.cloud"; - - assertEquals(endpoint1 + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint1)); - assertEquals(endpoint2 + " points to c0 us-west-1", - List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint2)); - assertEquals(endpoint3 + " points to c1 in all regions", - List.of("lb-1--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-1--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint3)); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2); assertEquals("Routing policy count is equal to cluster count", numberOfDeployments * clustersPerZone, tester.policiesOf(context1.instance().id()).size()); @@ -100,12 +93,10 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3); context1.submit(applicationPackage2).deploy(); - // Endpoint is updated to contain cluster in new deployment - assertEquals(endpoint1 + " points to c0 in all regions", - List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app1:default--prod.us-east-3/dns-zone-1/prod.us-east-3", - "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint1)); + // Endpoints are updated to contain cluster in new deployment + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); // Another application is deployed with a single cluster and global endpoint var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; @@ -116,10 +107,7 @@ public class RoutingPoliciesTest { .endpoint("r0", "c0") .build(); context2.submit(applicationPackage3).deploy(); - assertEquals(endpoint4 + " points to c0 in all regions", - List.of("lb-0--tenant1:app2:default--prod.us-central-1/dns-zone-1/prod.us-central-1", - "lb-0--tenant1:app2:default--prod.us-west-1/dns-zone-1/prod.us-west-1"), - tester.aliasDataOf(endpoint4)); + tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); // All endpoints for app1 are removed ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder() @@ -129,10 +117,10 @@ public class RoutingPoliciesTest { .allow(ValidationId.globalEndpointChange) .build(); context1.submit(applicationPackage4).deploy(); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint1)); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint2)); - assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint3)); - Set<RoutingPolicy> policies = tester.policiesOf(context1.instanceId()); + tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0); + tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0); + var policies = tester.policiesOf(context1.instanceId()); assertEquals(clustersPerZone * numberOfDeployments, policies.size()); assertTrue("Rotation membership is removed from all policies", policies.stream().allMatch(policy -> policy.endpoints().isEmpty())); @@ -226,8 +214,8 @@ public class RoutingPoliciesTest { "c1.app1.tenant1.us-central-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); - assertTrue("Removes stale routing policies " + context2.application(), tester.controllerTester().controller().applications().routingPolicies().get(context2.instanceId()).isEmpty()); - assertEquals("Keeps routing policies for " + context1.application(), 4, tester.controllerTester().controller().applications().routingPolicies().get(context1.instanceId()).size()); + assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty()); + assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size()); } @Test @@ -348,6 +336,145 @@ public class RoutingPoliciesTest { assertEquals("CNAME points to current load blancer", newHostname.value() + ".", tester.cnameDataOf(expectedRecords.iterator().next()).get(0)); } + + @Test + public void set_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Provision load balancers and deploy application + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage).deploy(); + + // Global DNS record is created + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + // Global routing status is overridden in one zone + var changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out, + GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + + // Inactive zone is removed from global DNS record + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Status details is stored in policy + var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Other zone remains in + var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent()); + assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt()); + + // Next deployment does not affect status + context.submit(applicationPackage).deploy(); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2); + + // Deployment is set back in + tester.controllerTester().clock().advance(Duration.ofHours(1)); + changedAt = tester.controllerTester().clock().instant(); + tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + + policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next(); + assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status()); + assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt()); + + // Deployment is set out through a new deployment.xml + var applicationPackage2 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region(), false) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage2).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1); + + // ... back in + var applicationPackage3 = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("r0", "c0", zone1.region().value(), zone2.region().value()) + .endpoint("r1", "c0", zone1.region().value(), zone2.region().value()) + .build(); + context.submit(applicationPackage3).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2); + } + + @Test + public void set_zone_global_endpoint_status() { + var tester = new RoutingPoliciesTester(); + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + var context2 = tester.newDeploymentContext("tenant2", "app2", "default"); + var contexts = List.of(context1, context2); + + // Deploy applications + var applicationPackage = new ApplicationPackageBuilder() + .region(zone1.region()) + .region(zone2.region()) + .endpoint("default", "c0", zone1.region().value(), zone2.region().value()) + .build(); + for (var context : contexts) { + tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deploy(); + tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } + + // Set zone out + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + for (var context : contexts) { + var policies = tester.routingPolicies().get(context.instanceId()); + assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in, + policies.values().stream() + .map(RoutingPolicy::status) + .map(Status::globalRouting) + .map(GlobalRouting::status) + .allMatch(status -> status == GlobalRouting.Status.in)); + } + var changedAt = tester.controllerTester().clock().instant(); + var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2); + assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status()); + assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent()); + assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt()); + + // Setting status per deployment does not affect status as entire zone is out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1); + + // Set single deployment out + tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant); + context1.flushDnsUpdates(); + + // Set zone back in. Deployment set explicitly out, remains out, the rest are in + tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in); + context1.flushDnsUpdates(); + tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1); + tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2); + } private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) { List<LoadBalancer> loadBalancers = new ArrayList<>(); @@ -372,6 +499,10 @@ public class RoutingPoliciesTest { this(new DeploymentTester()); } + public RoutingPolicies routingPolicies() { + return tester.controllerTester().controller().applications().routingPolicies(); + } + public DeploymentContext newDeploymentContext(String tenant, String application, String instance) { return tester.newDeploymentContext(tenant, application, instance); } @@ -391,8 +522,8 @@ public class RoutingPoliciesTest { } } - private Set<RoutingPolicy> policiesOf(ApplicationId instance) { - return tester.controller().curator().readRoutingPolicies(instance); + private Collection<RoutingPolicy> policiesOf(ApplicationId instance) { + return tester.controller().curator().readRoutingPolicies(instance).values(); } private Set<String> recordNames() { @@ -416,6 +547,21 @@ public class RoutingPoliciesTest { .collect(Collectors.toList()); } + private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) { + var prefix = ""; + if (!endpointId.equals(EndpointId.defaultId())) { + prefix = endpointId.id() + "."; + } + var endpoint = prefix + application.application().value() + "." + application.tenant().value() + + ".global.vespa.oath.cloud"; + var zoneTargets = Arrays.stream(zone) + .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" + + z.value() + "/dns-zone-1/" + z.value()) + .collect(Collectors.toSet()); + assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets, + Set.copyOf(aliasDataOf(endpoint))); + } + } } |