diff options
Diffstat (limited to 'controller-server')
7 files changed, 204 insertions, 73 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index eedc94c729c..c3e1ff1dbf2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -41,6 +41,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.configserver.Deployment import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; import com.yahoo.vespa.hosted.controller.api.integration.configserver.NodeFilter; +import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationStore; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion; import com.yahoo.vespa.hosted.controller.api.integration.deployment.ArtifactRepository; @@ -677,9 +678,10 @@ public class ApplicationController { operatorCertificates = Stream.concat(operatorCertificates.stream(), testerCertificate.stream()).toList(); } Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); + List<DataplaneTokenVersions> dataplaneTokenVersions = controller.dataplaneTokenService().listTokens(application.tenant()); DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform, endpoints, endpointCertificateMetadata, dockerImageRepo, domain, - deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dryRun); + deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData); return new DeploymentDataAndResult(deploymentData, preparedApplication.deploymentResult()); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index c50cc6051e1..81362018939 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -13,6 +13,9 @@ import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; @@ -71,6 +74,7 @@ public class RoutingController { private final Controller controller; private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; + private final BooleanFlag createTokenEndpoint; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -78,6 +82,7 @@ public class RoutingController { this.rotationRepository = new RotationRepository(Objects.requireNonNull(rotationsConfig, "rotationsConfig must be non-null"), controller.applications(), controller.curator()); + this.createTokenEndpoint = Flags.ENABLE_DATAPLANE_PROXY.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -109,11 +114,12 @@ public class RoutingController { /** Read and return zone-scoped endpoints for given deployment */ public EndpointList readEndpointsOf(DeploymentId deployment) { + boolean addTokenEndpoint = createTokenEndpoint.with(FetchVector.Dimension.APPLICATION_ID, deployment.applicationId().serializedForm()).value(); Set<Endpoint> endpoints = new LinkedHashSet<>(); // To discover the cluster name for a zone-scoped endpoint, we need to read routing policies for (var policy : routingPolicies.read(deployment)) { RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(policy.id().zone()); - endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod)); + endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)); endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod)); } return EndpointList.copyOf(endpoints); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 80b52e0c7a4..7b2e8d0f4ed 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -45,10 +45,11 @@ public class Endpoint { private final Scope scope; private final boolean legacy; private final RoutingMethod routingMethod; + private boolean tokenEndpoint; private Endpoint(TenantAndApplicationId application, Optional<InstanceName> instanceName, EndpointId id, ClusterSpec.Id cluster, URI url, URI legacyRegionalUrl, List<Target> targets, Scope scope, Port port, boolean legacy, - RoutingMethod routingMethod, boolean certificateName) { + RoutingMethod routingMethod, boolean certificateName, boolean tokenEndpoint) { Objects.requireNonNull(application, "application must be non-null"); Objects.requireNonNull(instanceName, "instanceName must be non-null"); Objects.requireNonNull(cluster, "cluster must be non-null"); @@ -66,6 +67,7 @@ public class Endpoint { this.scope = requireScope(scope, routingMethod); this.legacy = legacy; this.routingMethod = routingMethod; + this.tokenEndpoint = tokenEndpoint; } /** @@ -353,6 +355,10 @@ public class Endpoint { return targets; } + public boolean isTokenEndpoint() { + return tokenEndpoint; + } + /** An endpoint's scope */ public enum Scope { @@ -477,6 +483,7 @@ public class Endpoint { private RoutingMethod routingMethod = RoutingMethod.sharedLayer4; private boolean legacy = false; private boolean certificateName = false; + private boolean tokenEndpoint = false; private EndpointBuilder(TenantAndApplicationId application, Optional<InstanceName> instance) { this.application = Objects.requireNonNull(application); @@ -544,6 +551,11 @@ public class Endpoint { return this; } + public EndpointBuilder tokenEndpoint() { + this.tokenEndpoint = true; + return this; + } + /** Sets the port of this */ public EndpointBuilder on(Port port) { this.port = port; @@ -576,7 +588,8 @@ public class Endpoint { if (routingMethod.isDirect() && !port.isDefault()) { throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port"); } - URI url = createUrl(endpointOrClusterAsString(endpointId, cluster), + String prefix = tokenEndpoint ? "token-" : ""; + URI url = createUrl(prefix + endpointOrClusterAsString(endpointId, cluster), Objects.requireNonNull(application, "application must be non-null"), Objects.requireNonNull(instance, "instance must be non-null"), Objects.requireNonNull(targets, "targets must be non-null"), @@ -604,7 +617,8 @@ public class Endpoint { port, legacy, routingMethod, - certificateName); + certificateName, + tokenEndpoint); } private Scope requireUnset(Scope scope) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index 7a4d9edf66c..cd632917842 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -7,6 +7,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.FetchVector; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.identifiers.ClusterId; @@ -43,6 +46,8 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; /** @@ -53,12 +58,16 @@ import java.util.stream.Collectors; */ public class RoutingPolicies { + private static final Logger LOG = Logger.getLogger(RoutingPolicies.class.getName()); + private final Controller controller; private final CuratorDb db; + private final BooleanFlag createTokenEndpoint; public RoutingPolicies(Controller controller) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); this.db = controller.curator(); + this.createTokenEndpoint = Flags.ENABLE_DATAPLANE_PROXY.bindTo(controller.flagSource()); try (var lock = db.lockRoutingPolicies()) { // Update serialized format for (var policy : db.readRoutingPolicies().entrySet()) { db.writeRoutingPolicies(policy.getKey(), policy.getValue()); @@ -122,8 +131,8 @@ public class RoutingPolicies { instancePolicies = removePoliciesUnreferencedBy(allocation, instancePolicies, lock); applicationPolicies = applicationPolicies.replace(instance, instancePolicies); - updateGlobalDnsOf(instancePolicies, inactiveZones, owner, lock); - updateApplicationDnsOf(applicationPolicies, inactiveZones, owner, lock); + updateGlobalDnsOf(instancePolicies, Optional.of(deployment), inactiveZones, owner, lock); + updateApplicationDnsOf(applicationPolicies, inactiveZones, deployment, owner, lock); } } @@ -134,7 +143,7 @@ public class RoutingPolicies { controller.clock().instant()))); Map<ApplicationId, RoutingPolicyList> allPolicies = readAll().groupingBy(policy -> policy.id().owner()); allPolicies.forEach((instance, policies) -> { - updateGlobalDnsOf(policies, Set.of(), Optional.of(TenantAndApplicationId.from(instance)), lock); + updateGlobalDnsOf(policies, Optional.empty(), Set.of(), Optional.of(TenantAndApplicationId.from(instance)), lock); }); } } @@ -150,36 +159,64 @@ public class RoutingPolicies { var newPolicy = policy.with(RoutingStatus.create(value, agent, controller.clock().instant())); updatedPolicies.put(policy.id(), newPolicy); } - RoutingPolicyList effectivePolicies = RoutingPolicyList.copyOf(updatedPolicies.values()); Map<ApplicationId, RoutingPolicyList> policiesByInstance = effectivePolicies.groupingBy(policy -> policy.id().owner()); - policiesByInstance.forEach((owner, instancePolicies) -> db.writeRoutingPolicies(owner, instancePolicies.asList())); policiesByInstance.forEach((ignored, instancePolicies) -> updateGlobalDnsOf(instancePolicies, + Optional.of(deployment), Set.of(), ownerOf(deployment), lock)); - updateApplicationDnsOf(effectivePolicies, Set.of(), ownerOf(deployment), lock); + updateApplicationDnsOf(effectivePolicies, Set.of(), deployment, ownerOf(deployment), lock); + policiesByInstance.forEach((owner, instancePolicies) -> db.writeRoutingPolicies(owner, instancePolicies.asList())); } } /** Update global DNS records for given policies */ - private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Set<ZoneId> inactiveZones, - Optional<TenantAndApplicationId> owner, @SuppressWarnings("unused") Mutex lock) { + private void updateGlobalDnsOf(RoutingPolicyList instancePolicies, Optional<DeploymentId> deployment, + Set<ZoneId> inactiveZones, Optional<TenantAndApplicationId> owner, + @SuppressWarnings("unused") Mutex lock) { Map<RoutingId, List<RoutingPolicy>> routingTable = instancePolicies.asInstanceRoutingTable(); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); controller.routing().readDeclaredEndpointsOf(routingId.instance()) .named(routingId.endpointId(), Endpoint.Scope.global) .not().requiresRotation() - .forEach(endpoint -> updateGlobalDnsOf(endpoint, inactiveZones, routeEntry.getValue(), owner)); + .forEach(endpoint -> updateGlobalDnsOf(endpoint, inactiveZones, routeEntry.getValue(), deployment, owner)); } } /** Update global DNS records for given global endpoint */ - private void updateGlobalDnsOf(Endpoint endpoint, Set<ZoneId> inactiveZones, List<RoutingPolicy> policies, Optional<TenantAndApplicationId> owner) { + private void updateGlobalDnsOf(Endpoint endpoint, Set<ZoneId> inactiveZones, List<RoutingPolicy> policies, + Optional<DeploymentId> deployment, Optional<TenantAndApplicationId> owner) { if (endpoint.scope() != Endpoint.Scope.global) throw new IllegalArgumentException("Endpoint " + endpoint + " is not global"); - // Create a weighted ALIAS per region, pointing to all zones within the same region + if (deployment.isPresent() && !endpoint.deployments().contains(deployment.get())) return; + Collection<RegionEndpoint> regionEndpoints = computeRegionEndpoints(policies, inactiveZones); + Set<AliasTarget> latencyTargets = new LinkedHashSet<>(); + Set<AliasTarget> inactiveLatencyTargets = new LinkedHashSet<>(); + for (var regionEndpoint : regionEndpoints) { + if (regionEndpoint.active()) { + latencyTargets.add(regionEndpoint.target()); + } else { + inactiveLatencyTargets.add(regionEndpoint.target()); + } + } + + // Refuse removal of last target in an endpoint. We do this because removing 100% of the ALIAS records would + // cause the application endpoint to stop resolving entirely (NXDOMAIN). + if (latencyTargets.isEmpty() && !inactiveLatencyTargets.isEmpty()) { + if (deployment.isPresent()) { + throw new IllegalArgumentException("Cannot deactivate routing for " + deployment.get() + + " as it's the last remaining active deployment in " + endpoint); + } else { + // Operator is deactivating routing for entire zone, but this endpoint only has one target + LOG.log(Level.WARNING, "Cannot deactivate routing for " + endpoint + " because it has only one " + + "active zone. Leaving it in"); + return; + } + } + + // Create a weighted ALIAS per region, pointing to all zones within the same region regionEndpoints.forEach(regionEndpoint -> { if ( ! regionEndpoint.zoneAliasTargets().isEmpty()) { controller.nameServiceForwarder().createAlias(RecordName.from(regionEndpoint.target().name().value()), @@ -196,23 +233,6 @@ public class RoutingPolicies { }); // Create global latency-based ALIAS pointing to each per-region weighted ALIAS - Set<AliasTarget> latencyTargets = new LinkedHashSet<>(); - Set<AliasTarget> inactiveLatencyTargets = new LinkedHashSet<>(); - for (var regionEndpoint : regionEndpoints) { - if (regionEndpoint.active()) { - latencyTargets.add(regionEndpoint.target()); - } else { - inactiveLatencyTargets.add(regionEndpoint.target()); - } - } - - // If all targets are configured OUT, all targets are kept IN. We do this because otherwise removing 100% of - // the ALIAS records would cause the global endpoint to stop resolving entirely (NXDOMAIN). - if (latencyTargets.isEmpty() && !inactiveLatencyTargets.isEmpty()) { - latencyTargets.addAll(inactiveLatencyTargets); - inactiveLatencyTargets.clear(); - } - controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), latencyTargets, Priority.normal, owner); inactiveLatencyTargets.forEach(t -> controller.nameServiceForwarder() .removeRecords(Record.Type.ALIAS, @@ -254,7 +274,8 @@ public class RoutingPolicies { private void updateApplicationDnsOf(RoutingPolicyList routingPolicies, Set<ZoneId> inactiveZones, - Optional<TenantAndApplicationId> owner, @SuppressWarnings("unused") Mutex lock) { + DeploymentId deployment, Optional<TenantAndApplicationId> owner, + @SuppressWarnings("unused") Mutex lock) { // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e. // there is no scenario where more than one deployment within an instance can be a member the same // application-level endpoint. However, to allow this in the future the routing table remains @@ -281,8 +302,7 @@ public class RoutingPolicies { Set<Target> inactiveTargets = inactiveTargetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>()); if (isConfiguredOut(zonePolicy, policy, inactiveZones)) { inactiveTargets.add(Target.weighted(policy, target)); - } - else { + } else { activeTargets.add(Target.weighted(policy, target)); } } @@ -290,39 +310,49 @@ public class RoutingPolicies { } } - // If all targets are configured OUT, all targets are kept IN. We do this because otherwise removing 100% of - // the ALIAS records would cause the application endpoint to stop resolving entirely (NXDOMAIN). + // Refuse removal of last target in an endpoint. We do this because removing 100% of the ALIAS records would + // cause the application endpoint to stop resolving entirely (NXDOMAIN). targetsByEndpoint.forEach((endpoint, targets) -> { - if (targets.isEmpty()) targets.addAll(inactiveTargetsByEndpoint.remove(endpoint)); + if (targets.isEmpty()) { + throw new IllegalArgumentException("Cannot deactivate routing for " + deployment + + " as it's the last remaining active deployment in " + endpoint); + } }); + // Create DNS records for active targets targetsByEndpoint.forEach((applicationEndpoint, targets) -> { // Where multiple zones are permitted, they all have the same routing policy, and nameServiceForwarder (below). ZoneId targetZone = applicationEndpoint.targets().iterator().next().deployment().zoneId(); Set<AliasTarget> aliasTargets = new LinkedHashSet<>(); Set<DirectTarget> directTargets = new LinkedHashSet<>(); for (Target target : targets) { - if (target.aliasOrDirectTarget() instanceof AliasTarget at) aliasTargets.add(at); - else directTargets.add((DirectTarget) target.aliasOrDirectTarget()); + if (!target.deployment().equals(deployment)) continue; // Do not update target not matching this deployment + if (target.aliasOrDirectTarget() instanceof AliasTarget at) { + aliasTargets.add(at); + } else { + directTargets.add((DirectTarget) target.aliasOrDirectTarget()); + } } - - if ( ! aliasTargets.isEmpty()) { + if (!aliasTargets.isEmpty()) { nameServiceForwarderIn(targetZone).createAlias( RecordName.from(applicationEndpoint.dnsName()), aliasTargets, Priority.normal, owner); nameServiceForwarderIn(targetZone).removeRecords(Type.ALIAS, RecordName.from(applicationEndpoint.legacyRegionalDnsName()), Priority.normal, owner); } - if ( ! directTargets.isEmpty()) { + if (!directTargets.isEmpty()) { nameServiceForwarderIn(targetZone).createDirect( RecordName.from(applicationEndpoint.dnsName()), directTargets, Priority.normal, owner); nameServiceForwarderIn(targetZone).removeRecords(Type.DIRECT, RecordName.from(applicationEndpoint.legacyRegionalDnsName()), Priority.normal, owner); } }); + + // Remove DNS records for inactive targets inactiveTargetsByEndpoint.forEach((applicationEndpoint, targets) -> { // Where multiple zones are permitted, they all have the same routing policy, and nameServiceForwarder. ZoneId targetZone = applicationEndpoint.targets().iterator().next().deployment().zoneId(); targets.forEach(target -> { + if (!target.deployment().equals(deployment)) return; // Do not update target not matching this deployment nameServiceForwarderIn(targetZone).removeRecords(target.type(), RecordName.from(applicationEndpoint.dnsName()), target.data(), @@ -369,7 +399,8 @@ public class RoutingPolicies { /** Update zone DNS record for given policy */ private void updateZoneDnsOf(RoutingPolicy policy, LoadBalancer loadBalancer, DeploymentId deploymentId) { - for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive)) { + boolean addTokenEndpoint = createTokenEndpoint.with(FetchVector.Dimension.APPLICATION_ID, deploymentId.applicationId().serializedForm()).value(); + for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, addTokenEndpoint)) { var name = RecordName.from(endpoint.dnsName()); var record = policy.canonicalName().isPresent() ? new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : @@ -381,6 +412,7 @@ public class RoutingPolicies { private void setPrivateDns(Endpoint endpoint, LoadBalancer loadBalancer, DeploymentId deploymentId) { if (loadBalancer.service().isEmpty()) return; + if (endpoint.isTokenEndpoint()) return; controller.serviceRegistry().vpcEndpointService() .setPrivateDns(DomainName.of(endpoint.dnsName()), new ClusterId(deploymentId, endpoint.cluster()), @@ -437,12 +469,13 @@ public class RoutingPolicies { * @return the updated policies */ private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { + boolean addTokenEndpoint = createTokenEndpoint.with(FetchVector.Dimension.APPLICATION_ID, allocation.deployment.applicationId().serializedForm()).value(); Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) .not().matching(policy -> activeIds.contains(policy.id())); for (var policy : removable) { - for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive)) { + for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, addTokenEndpoint)) { nameServiceForwarderIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), Priority.normal, @@ -676,16 +709,16 @@ public class RoutingPolicies { } /** Denotes record data (record rhs) of either an ALIAS or a DIRECT target */ - private record Target(Record.Type type, RecordData data, Object aliasOrDirectTarget) { + private record Target(Record.Type type, RecordData data, DeploymentId deployment, Object aliasOrDirectTarget) { static Target weighted(RoutingPolicy policy, Endpoint.Target endpointTarget) { if (policy.ipAddress().isPresent()) { var wt = new WeightedDirectTarget(RecordData.from(policy.ipAddress().get()), endpointTarget.deployment().zoneId(), endpointTarget.weight()); - return new Target(Record.Type.DIRECT, wt.recordData(), wt); + return new Target(Record.Type.DIRECT, wt.recordData(), endpointTarget.deployment(), wt); } var wt = new WeightedAliasTarget(policy.canonicalName().get(), policy.dnsZone().get(), endpointTarget.deployment().zoneId().value(), endpointTarget.weight()); - return new Target(Record.Type.ALIAS, RecordData.fqdn(wt.name().value()), wt); + return new Target(Record.Type.ALIAS, RecordData.fqdn(wt.name().value()), endpointTarget.deployment(), wt); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index b4d83b7ded6..fb8f5e8e129 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -104,9 +104,15 @@ public record RoutingPolicy(RoutingPolicyId id, } /** Returns the zone endpoints of this */ - public List<Endpoint> zoneEndpointsIn(SystemName system, RoutingMethod routingMethod) { + public List<Endpoint> zoneEndpointsIn(SystemName system, RoutingMethod routingMethod, boolean includeTokenEndpoint) { DeploymentId deployment = new DeploymentId(id.owner(), id.zone()); - return List.of(endpoint(routingMethod).target(id.cluster(), deployment).in(system)); + Endpoint zoneEndpoint = endpoint(routingMethod).target(id.cluster(), deployment).in(system); + if (includeTokenEndpoint) { + Endpoint tokenEndpoint = endpoint(routingMethod).target(id.cluster(), deployment).tokenEndpoint().in(system); + return List.of(zoneEndpoint, tokenEndpoint); + } else { + return List.of(zoneEndpoint); + } } /** Returns the region endpoint of this */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 874d9468941..475da3224cd 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -1545,7 +1545,7 @@ public class ControllerTest { DeploymentId deployment = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"), Set.of(), Optional::empty, Optional.empty(), Optional.empty(), - Quota::unlimited, List.of(), List.of(), Optional::empty, false); + Quota::unlimited, List.of(), List.of(), Optional::empty, List.of(),false); tester.configServer().deploy(deploymentData); assertTrue(tester.configServer().application(deployment.applicationId(), deployment.zoneId()).isPresent()); tester.controller().applications().deactivate(deployment.applicationId(), deployment.zoneId()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 772877de8e3..0233db50ac6 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -16,6 +16,7 @@ import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; @@ -34,6 +35,8 @@ import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; +import com.yahoo.vespa.hosted.controller.dns.RemoveRecords; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import org.junit.jupiter.api.Test; @@ -50,12 +53,15 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collector; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; /** * @author mortent @@ -117,6 +123,16 @@ public class RoutingPoliciesTest { tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1); tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3); + // Ensure test deployment only updates endpoints of which it is a member + context1.submit(applicationPackage2) + .runJob(DeploymentContext.systemTest); + NameServiceQueue queue = tester.controllerTester().controller().curator().readNameServiceQueue(); + assertEquals(List.of(new RemoveRecords(Optional.of(TenantAndApplicationId.from(context1.instanceId())), + Record.Type.CNAME, + RecordName.from("app1.tenant1.us-east-1.test.vespa.oath.cloud"))), + queue.requests()); + context1.completeRollout(); + // Another application is deployed with a single cluster and global endpoint var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud"; tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2); @@ -323,6 +339,28 @@ public class RoutingPoliciesTest { } @Test + void zone_token_endpoints() { + var tester = new RoutingPoliciesTester(); + tester.enableTokenEndpoint(true); + + var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); + + // Deploy application + tester.provisionLoadBalancers(1, context1.instanceId(), false, zone1, zone2); + context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Deployment creates records and policies for all clusters in all zones + Set<String> expectedRecords = Set.of( + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "token-c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "token-c0.app1.tenant1.us-central-1.vespa.oath.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(2, tester.policiesOf(context1.instanceId()).size()); + } + + @Test void zone_routing_policies_without_dns_update() { var tester = new RoutingPoliciesTester(new DeploymentTester(), false); var context = tester.newDeploymentContext("tenant1", "app1", "default"); @@ -730,37 +768,47 @@ public class RoutingPoliciesTest { context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - // Setting other deployment out implicitly sets all deployments in. Weight is set to zero, but that has no - // impact on routing decisions when the weight sum is zero - tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, - RoutingStatus.Agent.tenant); + // Setting remaining deployment out is rejected + try { + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); + } catch (IllegalArgumentException e) { + assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive]", e.getMessage()); + } context.flushDnsUpdates(); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); - // One inactive deployment is put back in. Global DNS record now points to the only active deployment + // Inactive deployment is put back in. Global DNS record now points to all deployments tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); + + // One deployment is deactivated again + tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, + RoutingStatus.Agent.tenant); + context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - // Setting zone (containing active deployment) out puts all deployments in + // Operator deactivates routing for entire zone where deployment only has that zone activated. This does not + // change status for the deployment as it's the only one left tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.out); context.flushDnsUpdates(); assertEquals(RoutingStatus.Value.out, tester.routingPolicies().read(zone1).routingStatus().value()); - tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L)); - - // Setting zone back in removes the currently inactive deployment - tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.in); - context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1); - // Inactive deployment is set in + // Inactive deployment is set in which allows the zone-wide status to take effect tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.in, RoutingStatus.Agent.tenant); context.flushDnsUpdates(); for (var policy : tester.routingPolicies().read(context.instanceId())) { assertSame(RoutingStatus.Value.in, policy.routingStatus().value()); } + tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); + + // Zone-wide status is changed to in + tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.in); + context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2); } @@ -790,8 +838,15 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(2, mainInstance, zone); } + // Application endpoints are not created until production jobs run + betaContext.submit(applicationPackage) + .runJob(DeploymentContext.systemTest); + assertEquals(Set.of("beta.app1.tenant1.us-east-1.test.vespa.oath.cloud"), tester.recordNames()); + betaContext.runJob(DeploymentContext.stagingTest); + assertEquals(Set.of("beta.app1.tenant1.us-east-3.staging.vespa.oath.cloud"), tester.recordNames()); + // Deploy both instances - betaContext.submit(applicationPackage).deploy(); + betaContext.completeRollout(); // Application endpoint points to both instances with correct weights DeploymentId betaZone5 = betaContext.deploymentIdIn(zone5); @@ -845,6 +900,15 @@ public class RoutingPoliciesTest { .readDeclaredEndpointsOf(application) .named(EndpointId.of("a1"), Endpoint.Scope.application).isEmpty(), "Endpoint removed"); + + // Ensure test deployment only updates endpoint of which it is a member + betaContext.submit(applicationPackage) + .runJob(DeploymentContext.systemTest); + NameServiceQueue queue = tester.controllerTester().controller().curator().readNameServiceQueue(); + assertEquals(List.of(new RemoveRecords(Optional.of(TenantAndApplicationId.from(betaContext.instanceId())), + Record.Type.CNAME, + RecordName.from("beta.app1.tenant1.us-east-1.test.vespa.oath.cloud"))), + queue.requests()); } @Test @@ -890,15 +954,17 @@ public class RoutingPoliciesTest { // Changing routing status for remaining deployments adds back all deployments, because removing all deployments // puts all IN tester.routingPolicies().setRoutingStatus(betaZone1, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); - tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); - betaContext.flushDnsUpdates(); - tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, - Map.of(betaZone1, 2, - mainZone1, 8, - mainZone2, 9)); + try { + tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); + fail("Expected exception"); + } catch (IllegalArgumentException e) { + assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive]", + e.getMessage()); + } - // Activating main deployment allows us to deactivate the beta deployment + // Re-activating one zone allows us to take out another tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.in, RoutingStatus.Agent.tenant); + tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); betaContext.flushDnsUpdates(); tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(mainZone1, 8)); @@ -1068,6 +1134,10 @@ public class RoutingPoliciesTest { .toList(); } + void enableTokenEndpoint(boolean enabled) { + tester.controllerTester().flagSource().withBooleanFlag(Flags.ENABLE_DATAPLANE_PROXY.id(), enabled); + } + /** Assert that an application endpoint points to given targets and weights */ private void assertTargets(TenantAndApplicationId application, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map<DeploymentId, Integer> deploymentWeights) { |