aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorØyvind Grønnesby <oyving@verizonmedia.com>2020-01-21 15:29:05 +0100
committerGitHub <noreply@github.com>2020-01-21 15:29:05 +0100
commita8a1ec711376e8146a639cb860f8c4860f8f0215 (patch)
tree33cf6c1268cf06b98784fffa4772c343d1b7b6f9
parentdf15e99f9d098ad7a6f1672be8c7736ea6bcdd53 (diff)
parentc5bbc53a26674ed11ebc342e2f7470a7eb5342e6 (diff)
Merge pull request #11859 from vespa-engine/mpolden/routing-control
Support overriding global routing status of routing policies
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java8
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java233
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java29
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java68
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java44
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java21
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java85
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java)10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java288
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java (renamed from controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java)77
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java57
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java53
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java49
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java14
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java36
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java23
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java55
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java29
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java19
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java (renamed from controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java)218
23 files changed, 997 insertions, 432 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index dfc9574fcd7..82120f13b75 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller;
import com.google.common.collect.ImmutableList;
@@ -59,7 +59,7 @@ import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.vespa.hosted.controller.deployment.Versions;
import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
-import com.yahoo.vespa.hosted.controller.maintenance.RoutingPolicies;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import com.yahoo.vespa.hosted.controller.rotation.RotationLock;
import com.yahoo.vespa.hosted.controller.rotation.RotationRepository;
@@ -686,9 +686,9 @@ public class ApplicationController {
catch (RuntimeException e) {
log.log(Level.WARNING, "Failed to get endpoint information for " + id, e);
}
- return routingPolicies.get(id).stream()
+ return routingPolicies.get(id).values().stream()
.filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone)
- .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(),
+ .collect(Collectors.toUnmodifiableMap(policy -> policy.id().cluster(),
policy -> policy.endpointIn(controller.system()).url()));
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index b23d16767be..c8cfc8ac286 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.deployment;
import com.yahoo.component.Version;
@@ -531,7 +531,7 @@ public class JobController {
DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system()));
return controller.applications().getDeploymentEndpoints(testerId)
.stream().findAny()
- .or(() -> controller.applications().routingPolicies().get(testerId).stream()
+ .or(() -> controller.applications().routingPolicies().get(testerId).values().stream()
.findAny()
.map(policy -> policy.endpointIn(controller.system()).url()));
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java
deleted file mode 100644
index ee38b2c9516..00000000000
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java
+++ /dev/null
@@ -1,233 +0,0 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;
-
-import com.yahoo.config.application.api.DeploymentSpec;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.hosted.controller.Controller;
-import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
-import com.yahoo.vespa.hosted.controller.application.Endpoint;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.RoutingId;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
-import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
-import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.LinkedHashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.logging.Logger;
-import java.util.stream.Collectors;
-
-/**
- * Updates routing policies and their associated DNS records based on an deployment's load balancers.
- *
- * @author mortent
- * @author mpolden
- */
-public class RoutingPolicies {
-
- private static final Logger LOGGER = Logger.getLogger(RoutingPolicies.class.getName());
-
- private final Controller controller;
- private final CuratorDb db;
-
- public RoutingPolicies(Controller controller) {
- this.controller = Objects.requireNonNull(controller, "controller must be non-null");
- this.db = controller.curator();
- try (var lock = db.lockRoutingPolicies()) { // Update serialized format
- for (var policy : db.readRoutingPolicies().entrySet()) {
- db.writeRoutingPolicies(policy.getKey(), policy.getValue());
- }
- }
- }
-
- /** Read all known routing policies for given instance */
- public Set<RoutingPolicy> get(ApplicationId application) {
- return db.readRoutingPolicies(application);
- }
-
- /** Read all known routing policies for given deployment */
- public Set<RoutingPolicy> get(DeploymentId deployment) {
- return get(deployment.applicationId(), deployment.zoneId());
- }
-
- /** Read all known routing policies for given deployment */
- public Set<RoutingPolicy> get(ApplicationId application, ZoneId zone) {
- return db.readRoutingPolicies(application).stream()
- .filter(policy -> policy.zone().equals(zone))
- .collect(Collectors.toUnmodifiableSet());
- }
-
- /**
- * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if
- * load balancers for given application have changed.
- */
- public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) {
- if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return;
- var lbs = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer().getLoadBalancers(application, zone),
- deploymentSpec);
- try (var lock = db.lockRoutingPolicies()) {
- removeObsoleteEndpointsFromDns(lbs, lock);
- storePoliciesOf(lbs, lock);
- removeObsoletePolicies(lbs, lock);
- registerEndpointsInDns(lbs, lock);
- }
- }
-
- /** Create global endpoints for given route, if any */
- private void registerEndpointsInDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
- Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application));
-
- // Create DNS record for each routing ID
- for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
- Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().endpointId(),
- controller.system());
- Set<AliasTarget> targets = routeEntry.getValue()
- .stream()
- .filter(policy -> policy.dnsZone().isPresent())
- .map(policy -> new AliasTarget(policy.canonicalName(),
- policy.dnsZone().get(),
- policy.zone()))
- .collect(Collectors.toSet());
- controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal);
- }
- }
-
- /** Store routing policies for given route. Returns the persisted policies. */
- private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
- var policies = new LinkedHashSet<>(get(loadBalancers.application));
- for (LoadBalancer loadBalancer : loadBalancers.list) {
- var endpointIds = loadBalancers.endpointIdsOf(loadBalancer);
- var policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer, endpointIds);
- if (!policies.add(policy)) {
- // Update existing policy
- policies.remove(policy);
- policies.add(policy);
- }
- }
- db.writeRoutingPolicies(loadBalancers.application, policies);
- }
-
- /** Create a policy for given load balancer and register a CNAME for it */
- private RoutingPolicy createPolicy(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer,
- Set<EndpointId> endpointIds) {
- var routingPolicy = new RoutingPolicy(application, loadBalancer.cluster(), zone, loadBalancer.hostname(),
- loadBalancer.dnsZone(), endpointIds, isActive(loadBalancer));
- var name = RecordName.from(routingPolicy.endpointIn(controller.system()).dnsName());
- var data = RecordData.fqdn(loadBalancer.hostname().value());
- controller.nameServiceForwarder().createCname(name, data, Priority.normal);
- return routingPolicy;
- }
-
- /** Remove obsolete policies for given route and their CNAME records */
- private void removeObsoletePolicies(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
- var allPolicies = new LinkedHashSet<>(get(loadBalancers.application));
- var removalCandidates = new HashSet<>(allPolicies);
- var activeLoadBalancers = loadBalancers.list.stream()
- .map(LoadBalancer::hostname)
- .collect(Collectors.toSet());
- // Remove active load balancers and irrelevant zones from candidates
- removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) ||
- !policy.zone().equals(loadBalancers.zone));
- for (var policy : removalCandidates) {
- var dnsName = policy.endpointIn(controller.system()).dnsName();
- controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal);
- allPolicies.remove(policy);
- }
- db.writeRoutingPolicies(loadBalancers.application, allPolicies);
- }
-
- /** Remove unreferenced global endpoints for given route from DNS */
- private void removeObsoleteEndpointsFromDns(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
- var zonePolicies = get(loadBalancers.application, loadBalancers.zone);
- var removalCandidates = routingTableFrom(zonePolicies).keySet();
- var activeRoutingIds = routingIdsFrom(loadBalancers);
- removalCandidates.removeAll(activeRoutingIds);
- for (var id : removalCandidates) {
- var endpoint = RoutingPolicy.endpointOf(id.application(), id.endpointId(), controller.system());
- controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal);
- }
- }
-
- /** Compute routing IDs from given load balancers */
- private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) {
- Set<RoutingId> routingIds = new LinkedHashSet<>();
- for (var loadBalancer : loadBalancers.list) {
- for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) {
- routingIds.add(new RoutingId(loadBalancer.application(), endpointId));
- }
- }
- return Collections.unmodifiableSet(routingIds);
- }
-
- /** Compute a routing table from given policies */
- private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) {
- var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>();
- for (var policy : routingPolicies) {
- for (var rotation : policy.endpoints()) {
- var id = new RoutingId(policy.owner(), rotation);
- routingTable.putIfAbsent(id, new ArrayList<>());
- routingTable.get(id).add(policy);
- }
- }
- return routingTable;
- }
-
- private static boolean isActive(LoadBalancer loadBalancer) {
- switch (loadBalancer.state()) {
- case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early
- // as possible
- case active: return true;
- }
- return false;
- }
-
- /** Load balancers allocated to a deployment */
- private static class AllocatedLoadBalancers {
-
- private final ApplicationId application;
- private final ZoneId zone;
- private final List<LoadBalancer> list;
- private final DeploymentSpec deploymentSpec;
-
- private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers,
- DeploymentSpec deploymentSpec) {
- this.application = application;
- this.zone = zone;
- this.list = List.copyOf(loadBalancers);
- this.deploymentSpec = deploymentSpec;
- }
-
- /** Compute all endpoint IDs for given load balancer */
- private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) {
- if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints
- return Set.of();
- }
- var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance());
- if (instanceSpec.isEmpty()) {
- return Set.of();
- }
- return instanceSpec.get().endpoints().stream()
- .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value()))
- .filter(endpoint -> endpoint.regions().contains(zone.region()))
- .map(com.yahoo.config.application.api.Endpoint::endpointId)
- .map(EndpointId::of)
- .collect(Collectors.toSet());
- }
-
- }
-
-}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 22894a084b6..1a2ffc69249 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -1,4 +1,4 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.persistence;
import com.google.common.util.concurrent.UncheckedTimeoutException;
@@ -18,19 +18,21 @@ import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.ApplicationCertificate;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.auditlog.AuditLog;
import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.vespa.hosted.controller.deployment.Step;
import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
+import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy;
import com.yahoo.vespa.hosted.controller.tenant.Tenant;
import com.yahoo.vespa.hosted.controller.versions.ControllerVersion;
import com.yahoo.vespa.hosted.controller.versions.OsVersion;
import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
-import org.apache.zookeeper.data.Stat;
import java.io.IOException;
import java.io.UncheckedIOException;
@@ -80,6 +82,7 @@ public class CuratorDb {
private static final Path jobRoot = root.append("jobs");
private static final Path controllerRoot = root.append("controllers");
private static final Path routingPoliciesRoot = root.append("routingPolicies");
+ private static final Path zoneRoutingPoliciesRoot = root.append("zoneRoutingPolicies");
private static final Path applicationCertificateRoot = root.append("applicationCertificates");
private final StringSetSerializer stringSetSerializer = new StringSetSerializer();
@@ -93,6 +96,7 @@ public class CuratorDb {
private final OsVersionSerializer osVersionSerializer = new OsVersionSerializer();
private final OsVersionStatusSerializer osVersionStatusSerializer = new OsVersionStatusSerializer(osVersionSerializer, nodeVersionSerializer);
private final RoutingPolicySerializer routingPolicySerializer = new RoutingPolicySerializer();
+ private final ZoneRoutingPolicySerializer zoneRoutingPolicySerializer = new ZoneRoutingPolicySerializer(routingPolicySerializer);
private final AuditLogSerializer auditLogSerializer = new AuditLogSerializer();
private final NameServiceQueueSerializer nameServiceQueueSerializer = new NameServiceQueueSerializer();
@@ -485,19 +489,28 @@ public class CuratorDb {
// -------------- Routing policies ----------------------------------------
- public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) {
+ public void writeRoutingPolicies(ApplicationId application, Map<RoutingPolicyId, RoutingPolicy> policies) {
curator.set(routingPolicyPath(application), asJson(routingPolicySerializer.toSlime(policies)));
}
- public Map<ApplicationId, Set<RoutingPolicy>> readRoutingPolicies() {
+ public Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> readRoutingPolicies() {
return curator.getChildren(routingPoliciesRoot).stream()
.map(ApplicationId::fromSerializedForm)
.collect(Collectors.toUnmodifiableMap(Function.identity(), this::readRoutingPolicies));
}
- public Set<RoutingPolicy> readRoutingPolicies(ApplicationId application) {
+ public Map<RoutingPolicyId, RoutingPolicy> readRoutingPolicies(ApplicationId application) {
return readSlime(routingPolicyPath(application)).map(slime -> routingPolicySerializer.fromSlime(application, slime))
- .orElseGet(Collections::emptySet);
+ .orElseGet(Map::of);
+ }
+
+ public void writeZoneRoutingPolicy(ZoneRoutingPolicy policy) {
+ curator.set(zoneRoutingPolicyPath(policy.zone()), asJson(zoneRoutingPolicySerializer.toSlime(policy)));
+ }
+
+ public ZoneRoutingPolicy readZoneRoutingPolicy(ZoneId zone) {
+ return readSlime(zoneRoutingPolicyPath(zone)).map(data -> zoneRoutingPolicySerializer.fromSlime(zone, data))
+ .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS));
}
// -------------- Application web certificates ----------------------------
@@ -581,6 +594,8 @@ public class CuratorDb {
return routingPoliciesRoot.append(application.serializedForm());
}
+ private static Path zoneRoutingPolicyPath(ZoneId zone) { return zoneRoutingPoliciesRoot.append(zone.value()); }
+
private static Path nameServiceQueuePath() {
return root.append("nameServiceQueue");
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
index 54a3ef7551a..2429c5ee8c5 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.persistence;
import com.yahoo.config.provision.ApplicationId;
@@ -6,13 +6,20 @@ import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.slime.ArrayTraverser;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
+import com.yahoo.vespa.hosted.controller.routing.Status;
+import java.time.Instant;
import java.util.Collections;
+import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
-import java.util.Set;
+import java.util.Map;
/**
* Serializer and deserializer for a {@link RoutingPolicy}.
@@ -35,45 +42,64 @@ public class RoutingPolicySerializer {
private static final String zoneField = "zone";
private static final String dnsZoneField = "dnsZone";
private static final String rotationsField = "rotations";
- private static final String activeField = "active";
+ private static final String loadBalancerActiveField = "active";
+ private static final String globalRoutingField = "globalRouting";
+ private static final String agentField = "agent";
+ private static final String changedAtField = "changedAt";
+ private static final String statusField = "status";
- public Slime toSlime(Set<RoutingPolicy> routingPolicies) {
+ public Slime toSlime(Map<RoutingPolicyId, RoutingPolicy> routingPolicies) {
var slime = new Slime();
var root = slime.setObject();
var policyArray = root.setArray(routingPoliciesField);
- routingPolicies.forEach(policy -> {
+ routingPolicies.values().forEach(policy -> {
var policyObject = policyArray.addObject();
- policyObject.setString(clusterField, policy.cluster().value());
- policyObject.setString(zoneField, policy.zone().value());
+ policyObject.setString(clusterField, policy.id().cluster().value());
+ policyObject.setString(zoneField, policy.id().zone().value());
policyObject.setString(canonicalNameField, policy.canonicalName().value());
policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone));
var rotationArray = policyObject.setArray(rotationsField);
policy.endpoints().forEach(endpointId -> {
rotationArray.addString(endpointId.id());
});
- policyObject.setBool(activeField, policy.active());
+ policyObject.setBool(loadBalancerActiveField, policy.status().isActive());
+ globalRoutingToSlime(policy.status().globalRouting(), policyObject.setObject(globalRoutingField));
});
return slime;
}
- public Set<RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) {
- var policies = new LinkedHashSet<RoutingPolicy>();
+ public Map<RoutingPolicyId, RoutingPolicy> fromSlime(ApplicationId owner, Slime slime) {
+ var policies = new LinkedHashMap<RoutingPolicyId, RoutingPolicy>();
var root = slime.get();
var field = root.field(routingPoliciesField);
field.traverse((ArrayTraverser) (i, inspect) -> {
var endpointIds = new LinkedHashSet<EndpointId>();
inspect.field(rotationsField).traverse((ArrayTraverser) (j, endpointId) -> endpointIds.add(EndpointId.of(endpointId.asString())));
- var activeFieldInspector = inspect.field(activeField);
- // TODO(mpolden): Remove field presence check after January 2020
- boolean active = !activeFieldInspector.valid() || activeFieldInspector.asBool();
- policies.add(new RoutingPolicy(owner,
- ClusterSpec.Id.from(inspect.field(clusterField).asString()),
- ZoneId.from(inspect.field(zoneField).asString()),
- HostName.from(inspect.field(canonicalNameField).asString()),
- Serializers.optionalString(inspect.field(dnsZoneField)),
- endpointIds, active));
+ var id = new RoutingPolicyId(owner,
+ ClusterSpec.Id.from(inspect.field(clusterField).asString()),
+ ZoneId.from(inspect.field(zoneField).asString()));
+ policies.put(id, new RoutingPolicy(id,
+ HostName.from(inspect.field(canonicalNameField).asString()),
+ Serializers.optionalString(inspect.field(dnsZoneField)),
+ endpointIds,
+ new Status(inspect.field(loadBalancerActiveField).asBool(),
+ globalRoutingFromSlime(inspect.field(globalRoutingField)))));
});
- return Collections.unmodifiableSet(policies);
+ return Collections.unmodifiableMap(policies);
+ }
+
+ public void globalRoutingToSlime(GlobalRouting globalRouting, Cursor object) {
+ object.setString(statusField, globalRouting.status().name());
+ object.setString(agentField, globalRouting.agent().name());
+ object.setLong(changedAtField, globalRouting.changedAt().toEpochMilli());
+ }
+
+ public GlobalRouting globalRoutingFromSlime(Inspector object) {
+ if (!object.valid()) return GlobalRouting.DEFAULT_STATUS;
+ var status = GlobalRouting.Status.valueOf(object.field(statusField).asString());
+ var agent = GlobalRouting.Agent.valueOf(object.field(agentField).asString());
+ var changedAt = Serializers.optionalInstant(object.field(changedAtField)).orElse(Instant.EPOCH);
+ return new GlobalRouting(status, agent, changedAt);
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java
new file mode 100644
index 00000000000..6688d16ad14
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializer.java
@@ -0,0 +1,44 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy;
+
+import java.util.Objects;
+
+/**
+ * Serializer for {@link ZoneRoutingPolicy}.
+ *
+ * @author mpolden
+ */
+public class ZoneRoutingPolicySerializer {
+
+ // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one
+ // (and rewrite all nodes on startup), changes to the serialized format must be made
+ // such that what is serialized on version N+1 can be read by version N:
+ // - ADDING FIELDS: Always ok
+ // - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version.
+ // - CHANGING THE FORMAT OF A FIELD: Don't do it bro.
+
+ private static final String GLOBAL_ROUTING_FIELD = "globalRouting";
+
+ private final RoutingPolicySerializer routingPolicySerializer;
+
+ public ZoneRoutingPolicySerializer(RoutingPolicySerializer routingPolicySerializer) {
+ this.routingPolicySerializer = Objects.requireNonNull(routingPolicySerializer, "routingPolicySerializer must be non-null");
+ }
+
+ public ZoneRoutingPolicy fromSlime(ZoneId zone, Slime slime) {
+ var root = slime.get();
+ return new ZoneRoutingPolicy(zone, routingPolicySerializer.globalRoutingFromSlime(root.field(GLOBAL_ROUTING_FIELD)));
+ }
+
+ public Slime toSlime(ZoneRoutingPolicy policy) {
+ var slime = new Slime();
+ var root = slime.setObject();
+ routingPolicySerializer.globalRoutingToSlime(policy.globalRouting(), root.setObject(GLOBAL_ROUTING_FIELD));
+ return slime;
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 378013b5e6d..f6cf776cbfa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.restapi.application;
import ai.vespa.hosted.api.Signatures;
@@ -68,7 +68,6 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentCost;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatus;
@@ -804,9 +803,9 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
.forEach(globalEndpointUrls::add);
// Per-cluster endpoints. These are backed by load balancers.
- Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id());
+ var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values();
for (var policy : routingPolicies) {
- policy.rotationEndpointsIn(controller.system()).asList().stream()
+ policy.globalEndpointsIn(controller.system()).asList().stream()
.map(Endpoint::url)
.map(URI::toString)
.forEach(globalEndpointUrls::add);
@@ -929,10 +928,10 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
.ifPresent(rotation -> object.setString("rotationId", rotation.asString()));
// Per-cluster rotations
- Set<RoutingPolicy> routingPolicies = controller.applications().routingPolicies().get(instance.id());
- for (RoutingPolicy policy : routingPolicies) {
- if (!policy.active()) continue;
- policy.rotationEndpointsIn(controller.system()).asList().stream()
+ var routingPolicies = controller.applications().routingPolicies().get(instance.id()).values();
+ for (var policy : routingPolicies) {
+ if (!policy.status().isActive()) continue;
+ policy.globalEndpointsIn(controller.system()).asList().stream()
.map(Endpoint::url)
.map(URI::toString)
.forEach(globalRotationsArray::addString);
@@ -1043,11 +1042,11 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
// Add endpoint(s) defined by routing policies
var endpointArray = response.setArray("endpoints");
- for (var policy : controller.applications().routingPolicies().get(deploymentId)) {
- if (!policy.active()) continue;
+ for (var policy : controller.applications().routingPolicies().get(deploymentId).values()) {
+ if (!policy.status().isActive()) continue;
Cursor endpointObject = endpointArray.addObject();
Endpoint endpoint = policy.endpointIn(controller.system());
- endpointObject.setString("cluster", policy.cluster().value());
+ endpointObject.setString("cluster", policy.id().cluster().value());
endpointObject.setBool("tls", endpoint.tls());
endpointObject.setString("url", endpoint.url().toString());
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java
new file mode 100644
index 00000000000..1b2cf4a7896
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GlobalRouting.java
@@ -0,0 +1,85 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import java.time.Instant;
+import java.util.Objects;
+
+/**
+ * Represents the global routing status of a {@link RoutingPolicy} or {@link ZoneRoutingPolicy}. This contains the
+ * time global routing status was last changed and who changed it.
+ *
+ * This is immutable.
+ *
+ * @author mpolden
+ */
+public class GlobalRouting {
+
+ public static final GlobalRouting DEFAULT_STATUS = new GlobalRouting(Status.in, Agent.system, Instant.EPOCH);
+
+ private final Status status;
+ private final Agent agent;
+ private final Instant changedAt;
+
+ /** DO NOT USE. Public for serialization purposes */
+ public GlobalRouting(Status status, Agent agent, Instant changedAt) {
+ this.status = Objects.requireNonNull(status, "status must be non-null");
+ this.agent = Objects.requireNonNull(agent, "agent must be non-null");
+ this.changedAt = Objects.requireNonNull(changedAt, "changedAt must be non-null");
+ }
+
+ /** The current status of this */
+ public Status status() {
+ return status;
+ }
+
+ /** The agent who last changed this */
+ public Agent agent() {
+ return agent;
+ }
+
+ /** The time this was last changed */
+ public Instant changedAt() {
+ return changedAt;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ GlobalRouting that = (GlobalRouting) o;
+ return status == that.status &&
+ agent == that.agent &&
+ changedAt.equals(that.changedAt);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(status, agent, changedAt);
+ }
+
+ @Override
+ public String toString() {
+ return "status " + status + ", changed by " + agent + " @ " + changedAt;
+ }
+
+ public static GlobalRouting status(Status status, Agent agent, Instant instant) {
+ return new GlobalRouting(status, agent, instant);
+ }
+
+ // Used in serialization. Do not change.
+ public enum Status {
+ /** Status is determined by health checks **/
+ in,
+
+ /** Status is explicitly set to out */
+ out,
+ }
+
+ /** Agents that can change the state of global routing */
+ public enum Agent {
+ operator,
+ tenant,
+ system,
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
index 7b0ec3d27ba..5543d0ea0b7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
@@ -1,7 +1,8 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
import java.util.Objects;
@@ -42,4 +43,9 @@ public class RoutingId {
return Objects.hash(application, endpointId);
}
+ @Override
+ public String toString() {
+ return "routing id for " + endpointId + " of " + application;
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
new file mode 100644
index 00000000000..c05152e7795
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -0,0 +1,288 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Updates routing policies and their associated DNS records based on an deployment's load balancers.
+ *
+ * @author mortent
+ * @author mpolden
+ */
+public class RoutingPolicies {
+
+ private final Controller controller;
+ private final CuratorDb db;
+
+ public RoutingPolicies(Controller controller) {
+ this.controller = Objects.requireNonNull(controller, "controller must be non-null");
+ this.db = controller.curator();
+ try (var lock = db.lockRoutingPolicies()) { // Update serialized format
+ for (var policy : db.readRoutingPolicies().entrySet()) {
+ db.writeRoutingPolicies(policy.getKey(), policy.getValue());
+ }
+ }
+ }
+
+ /** Read all known routing policies for given instance */
+ public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application) {
+ return db.readRoutingPolicies(application);
+ }
+
+ /** Read all known routing policies for given deployment */
+ public Map<RoutingPolicyId, RoutingPolicy> get(DeploymentId deployment) {
+ return get(deployment.applicationId(), deployment.zoneId());
+ }
+
+ /** Read all known routing policies for given deployment */
+ public Map<RoutingPolicyId, RoutingPolicy> get(ApplicationId application, ZoneId zone) {
+ return db.readRoutingPolicies(application).entrySet()
+ .stream()
+ .filter(kv -> kv.getKey().zone().equals(zone))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ /**
+ * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if
+ * load balancers for given application have changed.
+ */
+ public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) {
+ if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return;
+ var loadBalancers = new AllocatedLoadBalancers(application, zone, controller.serviceRegistry().configServer()
+ .getLoadBalancers(application, zone),
+ deploymentSpec);
+ var inactiveZones = inactiveZones(application, deploymentSpec);
+ try (var lock = db.lockRoutingPolicies()) {
+ removeGlobalDnsUnreferencedBy(loadBalancers, lock);
+ storePoliciesOf(loadBalancers, lock);
+ removePoliciesUnreferencedBy(loadBalancers, lock);
+ updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock);
+ }
+ }
+
+ /** Set the status of all global endpoints in given zone */
+ public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) {
+ try (var lock = db.lockRoutingPolicies()) {
+ db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator,
+ controller.clock().instant())));
+ var allPolicies = db.readRoutingPolicies();
+ for (var applicationPolicies : allPolicies.values()) {
+ updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock);
+ }
+ }
+ }
+
+ /** Set the status of all global endpoints for given deployment */
+ public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) {
+ try (var lock = db.lockRoutingPolicies()) {
+ var policies = get(deployment.applicationId());
+ var newPolicies = new LinkedHashMap<>(policies);
+ for (var policy : policies.values()) {
+ if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone
+ var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent,
+ controller.clock().instant())));
+ newPolicies.put(policy.id(), newPolicy);
+ }
+ db.writeRoutingPolicies(deployment.applicationId(), newPolicies);
+ updateGlobalDnsOf(newPolicies.values(), Set.of(), lock);
+ }
+ }
+
+ /** Update global DNS record for given policies */
+ private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
+ // Create DNS record for each routing ID
+ var routingTable = routingTableFrom(routingPolicies);
+ for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
+ var targets = new LinkedHashSet<AliasTarget>();
+ var staleTargets = new LinkedHashSet<AliasTarget>();
+ for (var policy : routeEntry.getValue()) {
+ if (policy.dnsZone().isEmpty()) continue;
+ var target = new AliasTarget(policy.canonicalName(), policy.dnsZone().get(), policy.id().zone());
+ var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
+ // Remove target zone if global routing status is set out at:
+ // - zone level (ZoneRoutingPolicy)
+ // - deployment level (RoutingPolicy)
+ // - application package level (deployment.xml)
+ if (anyOut(zonePolicy.globalRouting(), policy.status().globalRouting()) ||
+ inactiveZones.contains(policy.id().zone())) {
+ staleTargets.add(target);
+ } else {
+ targets.add(target);
+ }
+ }
+ if (!targets.isEmpty()) {
+ var endpoint = RoutingPolicy.globalEndpointOf(routeEntry.getKey().application(),
+ routeEntry.getKey().endpointId(), controller.system());
+ controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal);
+ }
+ staleTargets.forEach(t -> controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, t.asData(), Priority.normal));
+ }
+ }
+
+ /** Store routing policies for given load balancers */
+ private void storePoliciesOf(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+ var policies = new LinkedHashMap<>(get(loadBalancers.application));
+ for (LoadBalancer loadBalancer : loadBalancers.list) {
+ var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), loadBalancers.zone);
+ var existingPolicy = policies.get(policyId);
+ var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.dnsZone(),
+ loadBalancers.endpointIdsOf(loadBalancer),
+ new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS));
+ // Preserve global routing status for existing policy
+ if (existingPolicy != null) {
+ newPolicy = newPolicy.with(newPolicy.status().with(existingPolicy.status().globalRouting()));
+ }
+ updateZoneDnsOf(newPolicy);
+ policies.put(newPolicy.id(), newPolicy);
+ }
+ db.writeRoutingPolicies(loadBalancers.application, policies);
+ }
+
+ /** Update zone DNS record for given policy */
+ private void updateZoneDnsOf(RoutingPolicy policy) {
+ var name = RecordName.from(policy.endpointIn(controller.system()).dnsName());
+ var data = RecordData.fqdn(policy.canonicalName().value());
+ controller.nameServiceForwarder().createCname(name, data, Priority.normal);
+ }
+
+ /** Remove policies and zone DNS records unreferenced by given load balancers */
+ private void removePoliciesUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+ var policies = get(loadBalancers.application);
+ var newPolicies = new LinkedHashMap<>(policies);
+ var activeLoadBalancers = loadBalancers.list.stream().map(LoadBalancer::hostname).collect(Collectors.toSet());
+ for (var policy : policies.values()) {
+ // Leave active load balancers and irrelevant zones alone
+ if (activeLoadBalancers.contains(policy.canonicalName()) ||
+ !policy.id().zone().equals(loadBalancers.zone)) continue;
+
+ var dnsName = policy.endpointIn(controller.system()).dnsName();
+ controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal);
+ newPolicies.remove(policy.id());
+ }
+ db.writeRoutingPolicies(loadBalancers.application, newPolicies);
+ }
+
+ /** Remove unreferenced global endpoints from DNS */
+ private void removeGlobalDnsUnreferencedBy(AllocatedLoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+ var zonePolicies = get(loadBalancers.application, loadBalancers.zone).values();
+ var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet());
+ var activeRoutingIds = routingIdsFrom(loadBalancers);
+ removalCandidates.removeAll(activeRoutingIds);
+ for (var id : removalCandidates) {
+ var endpoint = RoutingPolicy.globalEndpointOf(id.application(), id.endpointId(), controller.system());
+ controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal);
+ }
+ }
+
+ /** Compute routing IDs from given load balancers */
+ private static Set<RoutingId> routingIdsFrom(AllocatedLoadBalancers loadBalancers) {
+ Set<RoutingId> routingIds = new LinkedHashSet<>();
+ for (var loadBalancer : loadBalancers.list) {
+ for (var endpointId : loadBalancers.endpointIdsOf(loadBalancer)) {
+ routingIds.add(new RoutingId(loadBalancer.application(), endpointId));
+ }
+ }
+ return Collections.unmodifiableSet(routingIds);
+ }
+
+ /** Compute a routing table from given policies */
+ private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) {
+ var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>();
+ for (var policy : routingPolicies) {
+ for (var endpoint : policy.endpoints()) {
+ var id = new RoutingId(policy.id().owner(), endpoint);
+ routingTable.putIfAbsent(id, new ArrayList<>());
+ routingTable.get(id).add(policy);
+ }
+ }
+ return Collections.unmodifiableMap(routingTable);
+ }
+
+ private static boolean anyOut(GlobalRouting... globalRouting) {
+ return Arrays.stream(globalRouting)
+ .map(GlobalRouting::status)
+ .anyMatch(status -> status == GlobalRouting.Status.out);
+ }
+
+ private static boolean isActive(LoadBalancer loadBalancer) {
+ switch (loadBalancer.state()) {
+ case reserved: // Count reserved as active as we want callers (application API) to see the endpoint as early
+ // as possible
+ case active: return true;
+ }
+ return false;
+ }
+
+ /** Load balancers allocated to a deployment */
+ private static class AllocatedLoadBalancers {
+
+ private final ApplicationId application;
+ private final ZoneId zone;
+ private final List<LoadBalancer> list;
+ private final DeploymentSpec deploymentSpec;
+
+ private AllocatedLoadBalancers(ApplicationId application, ZoneId zone, List<LoadBalancer> loadBalancers,
+ DeploymentSpec deploymentSpec) {
+ this.application = application;
+ this.zone = zone;
+ this.list = List.copyOf(loadBalancers);
+ this.deploymentSpec = deploymentSpec;
+ }
+
+ /** Compute all endpoint IDs for given load balancer */
+ private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) {
+ if (zone.environment().isManuallyDeployed()) { // Manual deployments do not have any configurable endpoints
+ return Set.of();
+ }
+ var instanceSpec = deploymentSpec.instance(loadBalancer.application().instance());
+ if (instanceSpec.isEmpty()) {
+ return Set.of();
+ }
+ return instanceSpec.get().endpoints().stream()
+ .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value()))
+ .filter(endpoint -> endpoint.regions().contains(zone.region()))
+ .map(com.yahoo.config.application.api.Endpoint::endpointId)
+ .map(EndpointId::of)
+ .collect(Collectors.toSet());
+ }
+
+ }
+
+ /** Returns zones where global routing is declared inactive for instance through deploymentSpec */
+ private static Set<ZoneId> inactiveZones(ApplicationId instance, DeploymentSpec deploymentSpec) {
+ var instanceSpec = deploymentSpec.instance(instance.instance());
+ if (instanceSpec.isEmpty()) return Set.of();
+ return instanceSpec.get().zones().stream()
+ .filter(zone -> zone.environment().isProduction())
+ .filter(zone -> !zone.active())
+ .map(zone -> ZoneId.from(zone.environment(), zone.region().get()))
+ .collect(Collectors.toUnmodifiableSet());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index 80a62d94f2e..b1b6d1ae58a 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -1,60 +1,46 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.application;
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
import com.google.common.collect.ImmutableSortedSet;
import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.EndpointList;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
/**
- * Represents the DNS routing policy for a load balancer. A routing policy is uniquely identified by its owner, cluster
- * and zone.
+ * Represents the DNS routing policy for a {@link com.yahoo.vespa.hosted.controller.application.Deployment}.
*
* @author mortent
* @author mpolden
*/
public class RoutingPolicy {
- private final ApplicationId owner;
- private final ClusterSpec.Id cluster;
- private final ZoneId zone;
+ private final RoutingPolicyId id;
private final HostName canonicalName;
private final Optional<String> dnsZone;
private final Set<EndpointId> endpoints;
- private final boolean active;
+ private final Status status;
/** DO NOT USE. Public for serialization purposes */
- public RoutingPolicy(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone, HostName canonicalName,
- Optional<String> dnsZone, Set<EndpointId> endpoints, boolean active) {
- this.owner = Objects.requireNonNull(owner, "owner must be non-null");
- this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null");
- this.zone = Objects.requireNonNull(zone, "zone must be non-null");
+ public RoutingPolicy(RoutingPolicyId id, HostName canonicalName, Optional<String> dnsZone, Set<EndpointId> endpoints,
+ Status status) {
+ this.id = Objects.requireNonNull(id, "id must be non-null");
this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null");
this.dnsZone = Objects.requireNonNull(dnsZone, "dnsZone must be non-null");
this.endpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(endpoints, "endpoints must be non-null"));
- this.active = active;
+ this.status = Objects.requireNonNull(status, "status must be non-null");
}
- /** The application owning this */
- public ApplicationId owner() {
- return owner;
- }
-
- /** The zone this applies to */
- public ZoneId zone() {
- return zone;
- }
-
- /** The cluster this applies to */
- public ClusterSpec.Id cluster() {
- return cluster;
+ /** The ID of this */
+ public RoutingPolicyId id() {
+ return id;
}
/** The canonical name for this (rhs of a CNAME or ALIAS record) */
@@ -72,19 +58,24 @@ public class RoutingPolicy {
return endpoints;
}
- /** Returns whether this is active (the underlying load balancer is in an active state) */
- public boolean active() {
- return active;
+ /** Returns the status of this */
+ public Status status() {
+ return status;
+ }
+
+ /** Returns a copy of this with status set to given status */
+ public RoutingPolicy with(Status status) {
+ return new RoutingPolicy(id, canonicalName, dnsZone, endpoints, status);
}
/** Returns the endpoint of this */
public Endpoint endpointIn(SystemName system) {
- return Endpoint.of(owner).target(cluster, zone).on(Port.tls()).directRouting().in(system);
+ return Endpoint.of(id.owner()).target(id.cluster(), id.zone()).on(Port.tls()).directRouting().in(system);
}
- /** Returns rotation endpoints of this */
- public EndpointList rotationEndpointsIn(SystemName system) {
- return EndpointList.of(endpoints.stream().map(endpointId -> endpointOf(owner, endpointId, system)));
+ /** Returns global endpoints which this is a member of */
+ public EndpointList globalEndpointsIn(SystemName system) {
+ return EndpointList.of(endpoints.stream().map(endpointId -> globalEndpointOf(id.owner(), endpointId, system)));
}
@Override
@@ -92,25 +83,23 @@ public class RoutingPolicy {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
RoutingPolicy that = (RoutingPolicy) o;
- return owner.equals(that.owner) &&
- cluster.equals(that.cluster) &&
- zone.equals(that.zone);
+ return id.equals(that.id);
}
@Override
public int hashCode() {
- return Objects.hash(owner, cluster, zone);
+ return Objects.hash(id);
}
@Override
public String toString() {
return String.format("%s [rotations: %s%s], %s owned by %s, in %s", canonicalName, endpoints,
- dnsZone.map(z -> ", DNS zone: " + z).orElse(""), cluster, owner.toShortString(),
- zone.value());
+ dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(),
+ id.zone().value());
}
- /** Returns the endpoint of given rotation */
- public static Endpoint endpointOf(ApplicationId application, EndpointId endpointId, SystemName system) {
+ /** Creates a global endpoint for given application */
+ public static Endpoint globalEndpointOf(ApplicationId application, EndpointId endpointId, SystemName system) {
return Endpoint.of(application).named(endpointId).on(Port.tls()).directRouting().in(system);
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java
new file mode 100644
index 00000000000..06002e874f1
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicyId.java
@@ -0,0 +1,57 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.zone.ZoneId;
+
+import java.util.Objects;
+
+/**
+ * Unique identifier for a {@link RoutingPolicy}.
+ *
+ * @author mpolden
+ */
+public class RoutingPolicyId {
+
+ private final ApplicationId owner;
+ private final ClusterSpec.Id cluster;
+ private final ZoneId zone;
+
+ public RoutingPolicyId(ApplicationId owner, ClusterSpec.Id cluster, ZoneId zone) {
+ this.owner = Objects.requireNonNull(owner, "owner must be non-null");
+ this.cluster = Objects.requireNonNull(cluster, "cluster must be non-null");
+ this.zone = Objects.requireNonNull(zone, "zone must be non-null");
+ }
+
+ /** The application owning this */
+ public ApplicationId owner() {
+ return owner;
+ }
+
+ /** The zone this applies to */
+ public ZoneId zone() {
+ return zone;
+ }
+
+ /** The cluster this applies to */
+ public ClusterSpec.Id cluster() {
+ return cluster;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ RoutingPolicyId that = (RoutingPolicyId) o;
+ return owner.equals(that.owner) &&
+ cluster.equals(that.cluster) &&
+ zone.equals(that.zone);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(owner, cluster, zone);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java
new file mode 100644
index 00000000000..51e59c7cf4f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/Status.java
@@ -0,0 +1,53 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import java.util.Objects;
+
+/**
+ * Represents the status of a routing policy.
+ *
+ * This is immutable.
+ *
+ * @author mpolden
+ */
+public class Status {
+
+ private final boolean active;
+ private final GlobalRouting globalRouting;
+
+ /** DO NOT USE. Public for serialization purposes */
+ public Status(boolean active, GlobalRouting globalRouting) {
+ this.active = active;
+ this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null");
+ }
+
+ /** Returns whether this is considered active according to the load balancer status */
+ public boolean isActive() {
+ return active;
+ }
+
+ /** Return status of global routing */
+ public GlobalRouting globalRouting() {
+ return globalRouting;
+ }
+
+ /** Returns a copy of this with global routing changed */
+ public Status with(GlobalRouting globalRouting) {
+ return new Status(active, globalRouting);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Status status = (Status) o;
+ return active == status.active &&
+ globalRouting.equals(status.globalRouting);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(active, globalRouting);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java
new file mode 100644
index 00000000000..262cacd325e
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/ZoneRoutingPolicy.java
@@ -0,0 +1,49 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.yahoo.config.provision.zone.ZoneId;
+
+import java.util.Objects;
+
+/**
+ * Represents the DNS routing policy for a zone. This takes precedence over of an individual {@link RoutingPolicy}.
+ *
+ * This is immutable.
+ *
+ * @author mpolden
+ */
+public class ZoneRoutingPolicy {
+
+ private final ZoneId zone;
+ private final GlobalRouting globalRouting;
+
+ public ZoneRoutingPolicy(ZoneId zone, GlobalRouting globalRouting) {
+ this.zone = Objects.requireNonNull(zone, "zone must be non-null");
+ this.globalRouting = Objects.requireNonNull(globalRouting, "globalRouting must be non-null");
+ }
+
+ /** The zone this applies to */
+ public ZoneId zone() {
+ return zone;
+ }
+
+ /** The status of global routing */
+ public GlobalRouting globalRouting() {
+ return globalRouting;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ ZoneRoutingPolicy that = (ZoneRoutingPolicy) o;
+ return zone.equals(that.zone) &&
+ globalRouting.equals(that.globalRouting);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(zone, globalRouting);
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
index f722eb4f6bb..7c3c30738d6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/versions/VespaVersion.java
@@ -1,17 +1,13 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.versions;
import com.yahoo.component.Version;
-import com.yahoo.vespa.hosted.controller.Application;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.application.ApplicationList;
import com.yahoo.vespa.hosted.controller.application.InstanceList;
-import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
-import com.yahoo.vespa.hosted.controller.deployment.DeploymentStatusList;
import java.time.Instant;
import java.time.ZoneOffset;
-import java.util.stream.Collectors;
import static com.yahoo.config.application.api.DeploymentSpec.UpgradePolicy;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index dbeb96337d1..c83463bc1ea 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller;
import com.yahoo.component.Version;
@@ -356,7 +356,6 @@ public final class ControllerTester {
return application;
}
-
public void deploy(ApplicationId id, ZoneId zone) {
deploy(id, zone, new ApplicationPackage(new byte[0]));
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index e4b5e77b377..9b0706d184f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -101,13 +101,19 @@ public class ApplicationPackageBuilder {
}
public ApplicationPackageBuilder region(RegionName regionName) {
- return region(regionName.value());
+ return region(regionName, true);
}
public ApplicationPackageBuilder region(String regionName) {
- environmentBody.append(" <region active='true'>");
- environmentBody.append(regionName);
- environmentBody.append("</region>\n");
+ return region(RegionName.from(regionName), true);
+ }
+
+ public ApplicationPackageBuilder region(RegionName regionName, boolean active) {
+ environmentBody.append(" <region active='")
+ .append(active)
+ .append("'>")
+ .append(regionName.value())
+ .append("</region>\n");
return this;
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index 2d0b625dcb3..2792a59b523 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -1,10 +1,12 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.deployment;
import com.yahoo.component.Version;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.AthenzDomain;
import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.security.KeyAlgorithm;
import com.yahoo.security.KeyUtils;
@@ -26,9 +28,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint
import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGeneratorMock;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.application.Deployment;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
+import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
+import com.yahoo.vespa.hosted.controller.routing.Status;
import javax.security.auth.x500.X500Principal;
import java.math.BigInteger;
@@ -38,6 +45,7 @@ import java.security.cert.X509Certificate;
import java.time.Instant;
import java.util.Collections;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -57,8 +65,8 @@ import static org.junit.Assert.assertTrue;
*
* References to this should be acquired through {@link DeploymentTester#newDeploymentContext}.
*
- * Tester code that is not specific to deployments should be added to either {@link ControllerTester} or
- * {@link DeploymentTester} instead of this class.
+ * Tester code that is not specific to a single application's deployment context should be added to either
+ * {@link ControllerTester} or {@link DeploymentTester} instead of this class.
*
* @author mpolden
* @author jonmv
@@ -197,6 +205,28 @@ public class DeploymentContext {
return this;
}
+ /** Add a routing policy for this in given zone, with status set to active */
+ public DeploymentContext addRoutingPolicy(ZoneId zone, boolean active) {
+ return addRoutingPolicy(instanceId, zone, active);
+ }
+
+ /** Add a routing policy for tester instance of this in given zone, with status set to active */
+ public DeploymentContext addTesterRoutingPolicy(ZoneId zone, boolean active) {
+ return addRoutingPolicy(testerId.id(), zone, active);
+ }
+
+ private DeploymentContext addRoutingPolicy(ApplicationId instance, ZoneId zone, boolean active) {
+ var clusterId = "default" + (!active ? "-inactive" : "");
+ var id = new RoutingPolicyId(instance, ClusterSpec.Id.from(clusterId), zone);
+ var policies = new LinkedHashMap<>(tester.controller().curator().readRoutingPolicies(instance));
+ policies.put(id, new RoutingPolicy(id, HostName.from("lb-host"),
+ Optional.empty(),
+ Set.of(EndpointId.of("c0")),
+ new Status(active, GlobalRouting.DEFAULT_STATUS)));
+ tester.controller().curator().writeRoutingPolicies(instance, policies);
+ return this;
+ }
+
/** Submit given application package for deployment */
public DeploymentContext submit(ApplicationPackage applicationPackage) {
return submit(applicationPackage, defaultSourceRevision);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 51726035cb3..e052b967c31 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -1,11 +1,10 @@
-// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.deployment;
import com.google.common.collect.ImmutableList;
import com.yahoo.component.Version;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.provision.AthenzDomain;
-import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.config.provision.SystemName;
@@ -24,7 +23,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
import com.yahoo.vespa.hosted.controller.api.integration.deployment.TesterCloud;
import com.yahoo.vespa.hosted.controller.api.integration.stubs.MockMailer;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
import org.junit.Before;
import org.junit.Test;
@@ -43,20 +41,18 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
-import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.error;
import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.info;
import static com.yahoo.vespa.hosted.controller.api.integration.LogEntry.Type.warning;
-import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId;
import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.applicationPackage;
import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.publicCdApplicationPackage;
+import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTester.instanceId;
import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.failed;
import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.succeeded;
import static com.yahoo.vespa.hosted.controller.deployment.Step.Status.unfinished;
-import static java.util.Collections.emptySet;
import static java.util.Collections.singletonList;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
@@ -208,19 +204,8 @@ public class InternalStepRunnerTest {
tester.configServer().convergeServices(app.testerId().id(), JobType.systemTest.zone(system()));
assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
assertEquals(unfinished, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
-
- tester.controller().curator().writeRoutingPolicies(app.instanceId(), Set.of(new RoutingPolicy(app.instanceId(),
- ClusterSpec.Id.from("default"),
- JobType.systemTest.zone(system()),
- HostName.from("host"),
- Optional.empty(),
- emptySet(), true)));
- tester.controller().curator().writeRoutingPolicies(app.testerId().id(), Set.of(new RoutingPolicy(app.testerId().id(),
- ClusterSpec.Id.from("default"),
- JobType.systemTest.zone(system()),
- HostName.from("host"),
- Optional.empty(),
- emptySet(), true)));
+ app.addRoutingPolicy(JobType.systemTest.zone(system()), true);
+ app.addTesterRoutingPolicy(JobType.systemTest.zone(system()), true);
tester.runner().run();;
assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installReal));
assertEquals(succeeded, tester.jobs().last(app.instanceId(), JobType.systemTest).get().stepStatuses().get(Step.installTester));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
index 23355bd6033..c9ec5adc98c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
@@ -1,22 +1,26 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.persistence;
-import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableMap;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.config.SlimeUtils;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
+import com.yahoo.vespa.hosted.controller.routing.Status;
import org.junit.Test;
+import java.time.Instant;
import java.util.Iterator;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
/**
* @author mortent
@@ -29,41 +33,46 @@ public class RoutingPolicySerializerTest {
public void serialization() {
var owner = ApplicationId.defaultId();
var endpoints = Set.of(EndpointId.of("r1"), EndpointId.of("r2"));
- var policies = ImmutableSet.of(new RoutingPolicy(owner,
- ClusterSpec.Id.from("my-cluster1"),
- ZoneId.from("prod", "us-north-1"),
+ var id1 = new RoutingPolicyId(owner,
+ ClusterSpec.Id.from("my-cluster1"),
+ ZoneId.from("prod", "us-north-1"));
+ var id2 = new RoutingPolicyId(owner,
+ ClusterSpec.Id.from("my-cluster2"),
+ ZoneId.from("prod", "us-north-2"));
+ var policies = ImmutableMap.of(id1, new RoutingPolicy(id1,
HostName.from("long-and-ugly-name"),
Optional.of("zone1"),
- endpoints, true),
- new RoutingPolicy(owner,
- ClusterSpec.Id.from("my-cluster2"),
- ZoneId.from("prod", "us-north-2"),
+ endpoints, new Status(true, GlobalRouting.DEFAULT_STATUS)),
+ id2, new RoutingPolicy(id2,
HostName.from("long-and-ugly-name-2"),
Optional.empty(),
- endpoints, false));
+ endpoints, new Status(false,
+ new GlobalRouting(GlobalRouting.Status.out,
+ GlobalRouting.Agent.tenant,
+ Instant.ofEpochSecond(123)))));
var serialized = serializer.fromSlime(owner, serializer.toSlime(policies));
assertEquals(policies.size(), serialized.size());
- for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext();) {
+ for (Iterator<RoutingPolicy> it1 = policies.values().iterator(), it2 = serialized.values().iterator(); it1.hasNext();) {
var expected = it1.next();
var actual = it2.next();
- assertEquals(expected.owner(), actual.owner());
- assertEquals(expected.cluster(), actual.cluster());
- assertEquals(expected.zone(), actual.zone());
+ assertEquals(expected.id(), actual.id());
assertEquals(expected.canonicalName(), actual.canonicalName());
assertEquals(expected.dnsZone(), actual.dnsZone());
assertEquals(expected.endpoints(), actual.endpoints());
- assertEquals(expected.active(), actual.active());
+ assertEquals(expected.status(), actual.status());
}
}
+ // TODO(mpolden): Remove after January 2020
@Test
public void legacy_serialization() {
- var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\"," +
- "\"canonicalName\":\"lb-0\"," +
- "\"dnsZone\":\"dns-zone-id\",\"rotations\":[]}]}";
- var serialized = serializer.fromSlime(ApplicationId.defaultId(), SlimeUtils.jsonToSlime(json));
- assertTrue(serialized.iterator().next().active());
-
+ var json = "{\"routingPolicies\":[{\"cluster\":\"default\",\"zone\":\"prod.us-north-1\",\"canonicalName\":\"lb-host\",\"dnsZone\":\"dnsZoneId\",\"rotations\":[\"default\"],\"active\":true}]}";
+ var owner = ApplicationId.defaultId();
+ var serialized = serializer.fromSlime(owner, SlimeUtils.jsonToSlime(json));
+ var id = new RoutingPolicyId(owner, ClusterSpec.Id.from("default"), ZoneId.from("prod", "us-north-1"));
+ var expected = Map.of(id, new RoutingPolicy(id, HostName.from("lb-host"), Optional.of("dnsZoneId"),
+ Set.of(EndpointId.defaultId()), new Status(true, GlobalRouting.DEFAULT_STATUS)));
+ assertEquals(expected, serialized);
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java
new file mode 100644
index 00000000000..6a089c5e1b0
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java
@@ -0,0 +1,29 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.persistence;
+
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy;
+import org.junit.Test;
+
+import java.time.Instant;
+
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author mpolden
+ */
+public class ZoneRoutingPolicySerializerTest {
+
+ @Test
+ public void serialization() {
+ var serializer = new ZoneRoutingPolicySerializer(new RoutingPolicySerializer());
+ var zone = ZoneId.from("prod", "us-north-1");
+ var policy = new ZoneRoutingPolicy(zone,
+ GlobalRouting.status(GlobalRouting.Status.out, GlobalRouting.Agent.operator,
+ Instant.ofEpochMilli(123)));
+ var serialized = serializer.fromSlime(zone, serializer.toSlime(policy));
+ assertEquals(policy, serialized);
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 30be5d9b399..96681dc1c8b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -1,4 +1,4 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.restapi.application;
import ai.vespa.hosted.api.MultiPartStreamer;
@@ -11,7 +11,6 @@ import com.yahoo.config.provision.ApplicationName;
import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.zone.ZoneId;
@@ -52,8 +51,6 @@ import com.yahoo.vespa.hosted.controller.application.Change;
import com.yahoo.vespa.hosted.controller.application.ClusterInfo;
import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.athenz.HostedAthenzIdentities;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
@@ -1434,18 +1431,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
.region("us-west-1")
.build();
app.submit(applicationPackage).deploy();
- Set<RoutingPolicy> policies = Set.of(new RoutingPolicy(app.instanceId(),
- ClusterSpec.Id.from("default"),
- ZoneId.from(Environment.prod, RegionName.from("us-west-1")),
- HostName.from("lb-0-canonical-name"),
- Optional.of("dns-zone-1"), Set.of(EndpointId.of("c0")), true),
- // Inactive policy is not included
- new RoutingPolicy(app.instanceId(),
- ClusterSpec.Id.from("deleted-cluster"),
- ZoneId.from(Environment.prod, RegionName.from("us-west-1")),
- HostName.from("lb-1-canonical-name"),
- Optional.of("dns-zone-1"), Set.of(), false));
- tester.controller().curator().writeRoutingPolicies(app.instanceId(), policies);
+ app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), true);
+ app.addRoutingPolicy(ZoneId.from(Environment.prod, RegionName.from("us-west-1")), false);
// GET application
tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1", GET)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 1bb20296bd2..c0420c7b895 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -1,5 +1,5 @@
-// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.hosted.controller.maintenance;
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.ValidationId;
@@ -15,7 +15,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
-import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -24,7 +24,12 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
import org.junit.Test;
import java.net.URI;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -68,21 +73,9 @@ public class RoutingPoliciesTest {
// Creates alias records
context1.submit(applicationPackage).deploy();
- var endpoint1 = "r0.app1.tenant1.global.vespa.oath.cloud";
- var endpoint2 = "r1.app1.tenant1.global.vespa.oath.cloud";
- var endpoint3 = "r2.app1.tenant1.global.vespa.oath.cloud";
-
- assertEquals(endpoint1 + " points to c0 in all regions",
- List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1",
- "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
- tester.aliasDataOf(endpoint1));
- assertEquals(endpoint2 + " points to c0 us-west-1",
- List.of("lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
- tester.aliasDataOf(endpoint2));
- assertEquals(endpoint3 + " points to c1 in all regions",
- List.of("lb-1--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1",
- "lb-1--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
- tester.aliasDataOf(endpoint3));
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1);
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2);
assertEquals("Routing policy count is equal to cluster count",
numberOfDeployments * clustersPerZone,
tester.policiesOf(context1.instance().id()).size());
@@ -100,12 +93,10 @@ public class RoutingPoliciesTest {
tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3);
context1.submit(applicationPackage2).deploy();
- // Endpoint is updated to contain cluster in new deployment
- assertEquals(endpoint1 + " points to c0 in all regions",
- List.of("lb-0--tenant1:app1:default--prod.us-central-1/dns-zone-1/prod.us-central-1",
- "lb-0--tenant1:app1:default--prod.us-east-3/dns-zone-1/prod.us-east-3",
- "lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
- tester.aliasDataOf(endpoint1));
+ // Endpoints are updated to contain cluster in new deployment
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3);
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1);
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3);
// Another application is deployed with a single cluster and global endpoint
var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud";
@@ -116,10 +107,7 @@ public class RoutingPoliciesTest {
.endpoint("r0", "c0")
.build();
context2.submit(applicationPackage3).deploy();
- assertEquals(endpoint4 + " points to c0 in all regions",
- List.of("lb-0--tenant1:app2:default--prod.us-central-1/dns-zone-1/prod.us-central-1",
- "lb-0--tenant1:app2:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
- tester.aliasDataOf(endpoint4));
+ tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
// All endpoints for app1 are removed
ApplicationPackage applicationPackage4 = new ApplicationPackageBuilder()
@@ -129,10 +117,10 @@ public class RoutingPoliciesTest {
.allow(ValidationId.globalEndpointChange)
.build();
context1.submit(applicationPackage4).deploy();
- assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint1));
- assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint2));
- assertEquals("DNS records are removed", List.of(), tester.aliasDataOf(endpoint3));
- Set<RoutingPolicy> policies = tester.policiesOf(context1.instanceId());
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0);
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0);
+ tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0);
+ var policies = tester.policiesOf(context1.instanceId());
assertEquals(clustersPerZone * numberOfDeployments, policies.size());
assertTrue("Rotation membership is removed from all policies",
policies.stream().allMatch(policy -> policy.endpoints().isEmpty()));
@@ -226,8 +214,8 @@ public class RoutingPoliciesTest {
"c1.app1.tenant1.us-central-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
- assertTrue("Removes stale routing policies " + context2.application(), tester.controllerTester().controller().applications().routingPolicies().get(context2.instanceId()).isEmpty());
- assertEquals("Keeps routing policies for " + context1.application(), 4, tester.controllerTester().controller().applications().routingPolicies().get(context1.instanceId()).size());
+ assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty());
+ assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size());
}
@Test
@@ -348,6 +336,145 @@ public class RoutingPoliciesTest {
assertEquals("CNAME points to current load blancer", newHostname.value() + ".",
tester.cnameDataOf(expectedRecords.iterator().next()).get(0));
}
+
+ @Test
+ public void set_global_endpoint_status() {
+ var tester = new RoutingPoliciesTester();
+ var context = tester.newDeploymentContext("tenant1", "app1", "default");
+
+ // Provision load balancers and deploy application
+ tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+ var applicationPackage = new ApplicationPackageBuilder()
+ .region(zone1.region())
+ .region(zone2.region())
+ .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+ .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
+ .build();
+ context.submit(applicationPackage).deploy();
+
+ // Global DNS record is created
+ tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+ tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+
+ // Global routing status is overridden in one zone
+ var changedAt = tester.controllerTester().clock().instant();
+ tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
+ GlobalRouting.Agent.tenant);
+ context.flushDnsUpdates();
+
+ // Inactive zone is removed from global DNS record
+ tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
+ tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
+
+ // Status details is stored in policy
+ var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
+ assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status());
+ assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent());
+ assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
+
+ // Other zone remains in
+ var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next();
+ assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status());
+ assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent());
+ assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt());
+
+ // Next deployment does not affect status
+ context.submit(applicationPackage).deploy();
+ context.flushDnsUpdates();
+ tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
+ tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
+
+ // Deployment is set back in
+ tester.controllerTester().clock().advance(Duration.ofHours(1));
+ changedAt = tester.controllerTester().clock().instant();
+ tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+ context.flushDnsUpdates();
+ tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+ tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+
+ policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
+ assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status());
+ assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent());
+ assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
+
+ // Deployment is set out through a new deployment.xml
+ var applicationPackage2 = new ApplicationPackageBuilder()
+ .region(zone1.region())
+ .region(zone2.region(), false)
+ .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+ .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
+ .build();
+ context.submit(applicationPackage2).deploy();
+ tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
+ tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1);
+
+ // ... back in
+ var applicationPackage3 = new ApplicationPackageBuilder()
+ .region(zone1.region())
+ .region(zone2.region())
+ .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+ .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
+ .build();
+ context.submit(applicationPackage3).deploy();
+ tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+ tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+ }
+
+ @Test
+ public void set_zone_global_endpoint_status() {
+ var tester = new RoutingPoliciesTester();
+ var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
+ var context2 = tester.newDeploymentContext("tenant2", "app2", "default");
+ var contexts = List.of(context1, context2);
+
+ // Deploy applications
+ var applicationPackage = new ApplicationPackageBuilder()
+ .region(zone1.region())
+ .region(zone2.region())
+ .endpoint("default", "c0", zone1.region().value(), zone2.region().value())
+ .build();
+ for (var context : contexts) {
+ tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+ context.submit(applicationPackage).deploy();
+ tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
+ }
+
+ // Set zone out
+ tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out);
+ context1.flushDnsUpdates();
+ tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
+ tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
+ for (var context : contexts) {
+ var policies = tester.routingPolicies().get(context.instanceId());
+ assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in,
+ policies.values().stream()
+ .map(RoutingPolicy::status)
+ .map(Status::globalRouting)
+ .map(GlobalRouting::status)
+ .allMatch(status -> status == GlobalRouting.Status.in));
+ }
+ var changedAt = tester.controllerTester().clock().instant();
+ var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2);
+ assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status());
+ assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent());
+ assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt());
+
+ // Setting status per deployment does not affect status as entire zone is out
+ tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+ context1.flushDnsUpdates();
+ tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
+ tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
+
+ // Set single deployment out
+ tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
+ context1.flushDnsUpdates();
+
+ // Set zone back in. Deployment set explicitly out, remains out, the rest are in
+ tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in);
+ context1.flushDnsUpdates();
+ tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
+ tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
+ }
private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) {
List<LoadBalancer> loadBalancers = new ArrayList<>();
@@ -372,6 +499,10 @@ public class RoutingPoliciesTest {
this(new DeploymentTester());
}
+ public RoutingPolicies routingPolicies() {
+ return tester.controllerTester().controller().applications().routingPolicies();
+ }
+
public DeploymentContext newDeploymentContext(String tenant, String application, String instance) {
return tester.newDeploymentContext(tenant, application, instance);
}
@@ -391,8 +522,8 @@ public class RoutingPoliciesTest {
}
}
- private Set<RoutingPolicy> policiesOf(ApplicationId instance) {
- return tester.controller().curator().readRoutingPolicies(instance);
+ private Collection<RoutingPolicy> policiesOf(ApplicationId instance) {
+ return tester.controller().curator().readRoutingPolicies(instance).values();
}
private Set<String> recordNames() {
@@ -416,6 +547,21 @@ public class RoutingPoliciesTest {
.collect(Collectors.toList());
}
+ private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) {
+ var prefix = "";
+ if (!endpointId.equals(EndpointId.defaultId())) {
+ prefix = endpointId.id() + ".";
+ }
+ var endpoint = prefix + application.application().value() + "." + application.tenant().value() +
+ ".global.vespa.oath.cloud";
+ var zoneTargets = Arrays.stream(zone)
+ .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" +
+ z.value() + "/dns-zone-1/" + z.value())
+ .collect(Collectors.toSet());
+ assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets,
+ Set.copyOf(aliasDataOf(endpoint)));
+ }
+
}
}