diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-08-30 11:14:04 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-08-30 15:16:58 +0200 |
commit | 40da844d9ab6765c9102931acd81a56860aef927 (patch) | |
tree | 1acee3b2f4551dbe0b57f6d9f2d6fa469fedf927 /controller-server | |
parent | 773644c6cac7a00aaaef3096ee52d1d87969e998 (diff) |
Precompute zone endpoints on deployment
Diffstat (limited to 'controller-server')
16 files changed, 545 insertions, 291 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index d10328b01cb..e88a808ec03 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -13,6 +13,7 @@ import com.yahoo.config.provision.CloudAccount; import com.yahoo.config.provision.CloudName; import com.yahoo.config.provision.DockerImage; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.Tags; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.text.Text; @@ -28,6 +29,7 @@ import com.yahoo.vespa.flags.ListFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentEndpoints; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId; import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController; @@ -36,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult; import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult.LogEntry; import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node; @@ -55,13 +56,13 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning; import com.yahoo.vespa.hosted.controller.application.DeploymentQuotaCalculator; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator; +import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade; import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates; import com.yahoo.vespa.hosted.controller.concurrent.Once; @@ -72,6 +73,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Run; import com.yahoo.vespa.hosted.controller.notification.Notification; import com.yahoo.vespa.hosted.controller.notification.NotificationSource; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpoints; +import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant; @@ -89,7 +92,6 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; -import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -515,27 +517,34 @@ public class ApplicationController { RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision()); ApplicationPackageStream applicationPackage = new ApplicationPackageStream(() -> applicationStore.stream(deployment, revision)); AtomicReference<RevisionId> lastRevision = new AtomicReference<>(); - Instance instance; - Set<ContainerEndpoint> containerEndpoints; - try (Mutex lock = lock(applicationId)) { - LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); - application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); - instance = application.get().require(job.application().instance()); - - containerEndpoints = controller.routing().of(deployment).prepare(application); - } // Release application lock while doing the deployment, which is a lengthy task. - - Supplier<Optional<EndpointCertificate>> endpointCertificate = () -> { + Supplier<PreparedEndpoints> preparedEndpoints = () -> { try (Mutex lock = lock(applicationId)) { - Optional<EndpointCertificate> data = endpointCertificates.get(instance, zone, applicationPackage.truncatedPackage().deploymentSpec()); - data.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); - return data; + LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); + application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); + Instance instance = application.get().require(job.application().instance()); + Tags tags = applicationPackage.truncatedPackage().deploymentSpec().instance(job.application().instance()) + .map(DeploymentInstanceSpec::tags) + .orElseGet(Tags::empty); + Optional<EndpointCertificate> certificate = endpointCertificates.get(instance, zone, applicationPackage.truncatedPackage().deploymentSpec()); + certificate.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); + BasicServicesXml services; + try { + services = applicationPackage.truncatedPackage().services(deployment, tags); + } catch (Exception e) { + // If the basic parsing done by the controller fails, we ignore the exception here so that + // complete parsing errors are propagated from the config server. Otherwise, throwing here + // will interrupt the request while it's being streamed to the config server + log.warning("Ignoring failure to parse services.xml for deployment " + deployment + + " while streaming application package: " + Exceptions.toMessageString(e)); + services = BasicServicesXml.empty; + } + return controller.routing().of(deployment).prepare(services, certificate, application); } }; // Carry out deployment without holding the application lock. - DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints, - endpointCertificate, run.isDryRun(), run.testerCertificate()); + DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, preparedEndpoints, + run.isDryRun(), run.testerCertificate()); // Record the quota usage for this application @@ -635,7 +644,7 @@ public class ApplicationController { ApplicationPackageStream applicationPackage = new ApplicationPackageStream( () -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version)) ); - return deploy(application.id(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()).result(); + return deploy(application.id(), applicationPackage, zone, version, null, false, Optional.empty()).result(); } else { throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString()); } @@ -643,18 +652,18 @@ public class ApplicationController { /** Deploys the given tester application to the given zone. */ public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) { - return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()).result(); + return deploy(tester.id(), applicationPackage, zone, platform, null, false, Optional.empty()).result(); } private record DeploymentDataAndResult(DeploymentData data, DeploymentResult result) {} + private DeploymentDataAndResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage, - ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints, - Supplier<Optional<EndpointCertificate>> endpointCertificate, + ZoneId zone, Version platform, Supplier<PreparedEndpoints> preparedEndpoints, boolean dryRun, Optional<X509Certificate> testerCertificate) { DeploymentId deployment = new DeploymentId(application, zone); // Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails. interface CleanCloseable extends AutoCloseable { void close(); } - List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>(); + AtomicReference<GeneratedEndpoints> generatedEndpoints = new AtomicReference<>(GeneratedEndpoints.empty); try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage, generatedEndpoints)) { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag @@ -684,26 +693,23 @@ public class ApplicationController { } Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); List<DataplaneTokenVersions> dataplaneTokenVersions = controller.dataplaneTokenService().listTokens(application.tenant()); - Supplier<Optional<EndpointCertificate>> endpointCertificateWrapper = () -> { - Optional<EndpointCertificate> data = endpointCertificate.get(); - // TODO(mpolden): Pass these endpoints to config server as part of the deploy call. This will let the - // application know which endpoints are mTLS and which are token-based - data.flatMap(EndpointCertificate::randomizedId) - .ifPresent(applicationPart -> generatedEndpoints.addAll(controller.routing().generateEndpoints(applicationPart, deployment.applicationId()))); - return data; + Supplier<DeploymentEndpoints> endpoints = () -> { + if (preparedEndpoints == null) return DeploymentEndpoints.none; + PreparedEndpoints prepared = preparedEndpoints.get(); + generatedEndpoints.set(prepared.generatedEndpoints()); + return new DeploymentEndpoints(prepared.containerEndpoints(), prepared.certificate()); }; DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform, - endpoints, endpointCertificateWrapper, dockerImageRepo, domain, - deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun); + endpoints, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData); return new DeploymentDataAndResult(deploymentData, preparedApplication.deploymentResult()); } } - private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, List<GeneratedEndpoint> generatedEndpoints) { + private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, AtomicReference<GeneratedEndpoints> generatedEndpoints) { if (id.applicationId().instance().isTester()) return; - controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec(), generatedEndpoints); + controller.routing().of(id).activate(data.truncatedPackage().deploymentSpec(), generatedEndpoints.get()); if ( ! id.zoneId().environment().isManuallyDeployed()) return; controller.applications().applicationStore().putMeta(id, clock.instant(), data.truncatedPackage().metaDataZip()); } @@ -784,7 +790,7 @@ public class ApplicationController { throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments); for (Instance instance : application.get().instances().values()) { - controller.routing().removeEndpointsInDns(application.get(), instance.name()); + controller.routing().removeRotationEndpointsFromDns(application.get(), instance.name()); application = application.without(instance.name()); } @@ -820,7 +826,7 @@ public class ApplicationController { && application.get().deploymentSpec().instanceNames().contains(instanceId.instance())) throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first"); - controller.routing().removeEndpointsInDns(application.get(), instanceId.instance()); + controller.routing().removeRotationEndpointsFromDns(application.get(), instanceId.instance()); curator.writeApplication(application.without(instanceId.instance()).get()); controller.jobController().collectGarbage(); controller.notificationsDb().removeNotifications(NotificationSource.from(instanceId)); @@ -873,7 +879,7 @@ public class ApplicationController { /** * Asks the config server whether this deployment is currently healthy, i.e., serving traffic as usual. - * If this cannot be ascertained, we must assumed it is not. + * If this cannot be ascertained, we must assume it is not. */ public boolean isHealthy(DeploymentId deploymentId) { try { @@ -918,7 +924,7 @@ public class ApplicationController { DeploymentId id = new DeploymentId(instanceId, zone); interface CleanCloseable extends AutoCloseable { void close(); } try (CleanCloseable postDeactivation = () -> { - application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec(), List.of())); + application.ifPresent(app -> controller.routing().of(id).activate(app.get().deploymentSpec(), GeneratedEndpoints.empty)); if (id.zoneId().environment().isManuallyDeployed()) applicationStore.putMetaTombstone(id, clock.instant()); if ( ! id.zoneId().environment().isTest()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index d1f5d78bcdd..cc6195c075d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -18,7 +18,6 @@ import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; @@ -30,8 +29,10 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; -import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; +import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpoints; +import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; import com.yahoo.vespa.hosted.controller.routing.RoutingId; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies; import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext; @@ -52,13 +53,11 @@ import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.OptionalInt; import java.util.Set; import java.util.TreeMap; import java.util.stream.Collectors; @@ -118,22 +117,95 @@ public class RoutingController { return rotationRepository; } - /** Read and return zone-scoped endpoints for given deployment */ + /** Prepares and returns the endpoints relevant for given deployment */ + public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) { + EndpointList endpoints = EndpointList.EMPTY; + + // Assign rotations to application + for (var deploymentInstanceSpec : application.get().deploymentSpec().instances()) { + if (deploymentInstanceSpec.concerns(Environment.prod)) { + application = controller.routing().assignRotations(application, deploymentInstanceSpec.name()); + } + } + + // Add zone-scoped endpoints + final GeneratedEndpoints generatedEndpoints; + if (!usesSharedRouting(deployment.zoneId())) { // TODO(mpolden): Remove this check when config models < 8.230 are gone + boolean includeTokenEndpoint = tokenEndpointEnabled(deployment.applicationId()); + Map<ClusterSpec.Id, List<GeneratedEndpoint>> generatedEndpointsByCluster = new HashMap<>(); + for (var container : services.containers()) { + ClusterSpec.Id clusterId = ClusterSpec.Id.from(container.id()); + boolean tokenSupported = includeTokenEndpoint && container.authMethods().contains(BasicServicesXml.Container.AuthMethod.token); + List<GeneratedEndpoint> generatedForCluster = certificate.flatMap(EndpointCertificate::randomizedId) + .map(id -> generateEndpoints(id, deployment.applicationId(), tokenSupported)) + .orElseGet(List::of); + if (!generatedForCluster.isEmpty()) { + generatedEndpointsByCluster.put(clusterId, generatedForCluster); + } + endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedForCluster).scope(Scope.zone)); + } + generatedEndpoints = new GeneratedEndpoints(generatedEndpointsByCluster); + + } else { + generatedEndpoints = GeneratedEndpoints.empty; + } + + // Add global- and application-scoped endpoints + endpoints = endpoints.and(declaredEndpointsOf(application.get().id(), application.get().deploymentSpec(), generatedEndpoints).targets(deployment)); + PreparedEndpoints prepared = new PreparedEndpoints(deployment, + endpoints, + application.get().require(deployment.applicationId().instance()).rotations(), + certificate); + + // Register rotation-backed endpoints in DNS + registerRotationEndpointsInDns(prepared); + + return prepared; + } + + /** Read and return zone- and region-scoped endpoints for given deployment */ public EndpointList readEndpointsOf(DeploymentId deployment) { - boolean addTokenEndpoint = tokenEndpointEnabled(deployment.applicationId()); Set<Endpoint> endpoints = new LinkedHashSet<>(); - // To discover the cluster name for a zone-scoped endpoint, we need to read the routing policy for (var policy : routingPolicies.read(deployment)) { - RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(policy.id().zone()); - endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)); - endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod, Optional.empty())); - for (var ge : policy.generatedEndpoints()) { - boolean include = switch (ge.authMethod()) { - case token -> addTokenEndpoint; - case mtls -> true; - }; - if (include) { - endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod, Optional.of(ge))); + endpoints.addAll(endpointsOf(deployment, policy.id().cluster(), policy.generatedEndpoints()).asList()); + } + return EndpointList.copyOf(endpoints); + } + + /** Returns the zone- and region-scoped endpoints of given deployment */ + public EndpointList endpointsOf(DeploymentId deployment, ClusterSpec.Id cluster, List<GeneratedEndpoint> generatedEndpoints) { + // TODO(mpolden): Support tokens only when generated endpoints are available + boolean tokenSupported = tokenEndpointEnabled(deployment.applicationId()) && + (generatedEndpoints.isEmpty() || generatedEndpoints.stream().anyMatch(ge -> ge.authMethod() == AuthMethod.token)); + RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deployment.zoneId()); + boolean isProduction = deployment.zoneId().environment().isProduction(); + List<Endpoint> endpoints = new ArrayList<>(); + Endpoint.EndpointBuilder zoneEndpoint = Endpoint.of(deployment.applicationId()) + .routingMethod(routingMethod) + .on(Port.fromRoutingMethod(routingMethod)) + .target(cluster, deployment); + endpoints.add(zoneEndpoint.in(controller.system())); + if (tokenSupported) { + endpoints.add(zoneEndpoint.authMethod(AuthMethod.token).in(controller.system())); + } + Endpoint.EndpointBuilder regionEndpoint = Endpoint.of(deployment.applicationId()) + .routingMethod(routingMethod) + .on(Port.fromRoutingMethod(routingMethod)) + .targetRegion(cluster, deployment.zoneId()); + // Region endpoints are only used by global- and application-endpoints and are thus only needed in + // production environments + if (isProduction) { + endpoints.add(regionEndpoint.in(controller.system())); + } + for (var generatedEndpoint : generatedEndpoints) { + boolean include = switch (generatedEndpoint.authMethod()) { + case token -> tokenSupported; + case mtls -> true; + }; + if (include) { + endpoints.add(zoneEndpoint.generatedFrom(generatedEndpoint).in(controller.system())); + if (isProduction) { + endpoints.add(regionEndpoint.generatedFrom(generatedEndpoint).in(controller.system())); } } } @@ -148,43 +220,47 @@ public class RoutingController { /** Read application and return declared endpoints for given application */ public EndpointList readDeclaredEndpointsOf(TenantAndApplicationId application) { - return declaredEndpointsOf(controller.applications().requireApplication(application)); + return readDeclaredEndpointsOf(controller.applications().requireApplication(application)); + } + + public EndpointList readDeclaredEndpointsOf(Application application) { + return declaredEndpointsOf(application.id(), application.deploymentSpec(), readMultiDeploymentGeneratedEndpoints(application.id())); } /** Returns endpoints declared in {@link DeploymentSpec} for given application */ - public EndpointList declaredEndpointsOf(Application application) { - List<GeneratedEndpoint> generatedEndpoints = readGeneratedEndpoints(application); + private EndpointList declaredEndpointsOf(TenantAndApplicationId application, DeploymentSpec deploymentSpec, GeneratedEndpoints generatedEndpoints) { Set<Endpoint> endpoints = new LinkedHashSet<>(); - DeploymentSpec deploymentSpec = application.deploymentSpec(); + // Global endpoints for (var spec : deploymentSpec.instances()) { - ApplicationId instance = application.id().instance(spec.name()); - // Add endpoints declared with current syntax + ApplicationId instance = application.instance(spec.name()); spec.endpoints().forEach(declaredEndpoint -> { RoutingId routingId = RoutingId.of(instance, EndpointId.of(declaredEndpoint.endpointId())); List<DeploymentId> deployments = declaredEndpoint.regions().stream() .map(region -> new DeploymentId(instance, ZoneId.from(Environment.prod, region))) .toList(); - endpoints.addAll(computeGlobalEndpoints(routingId, ClusterSpec.Id.from(declaredEndpoint.containerId()), deployments, generatedEndpoints)); + ClusterSpec.Id cluster = ClusterSpec.Id.from(declaredEndpoint.containerId()); + endpoints.addAll(computeGlobalEndpoints(routingId, cluster, deployments, generatedEndpoints)); }); } - // Add application endpoints + // Application endpoints for (var declaredEndpoint : deploymentSpec.endpoints()) { Map<DeploymentId, Integer> deployments = declaredEndpoint.targets().stream() - .collect(toMap(t -> new DeploymentId(application.id().instance(t.instance()), + .collect(toMap(t -> new DeploymentId(application.instance(t.instance()), ZoneId.from(Environment.prod, t.region())), t -> t.weight())); ZoneId zone = deployments.keySet().iterator().next().zoneId(); // Where multiple zones are possible, they all have the same routing method. RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive; - Endpoint.EndpointBuilder builder = Endpoint.of(application.id()) + ClusterSpec.Id cluster = ClusterSpec.Id.from(declaredEndpoint.containerId()); + Endpoint.EndpointBuilder builder = Endpoint.of(application) .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), - ClusterSpec.Id.from(declaredEndpoint.containerId()), + cluster, deployments) .routingMethod(routingMethod) .on(Port.fromRoutingMethod(routingMethod)); endpoints.add(builder.in(controller.system())); - for (var ge : generatedEndpoints) { + for (var ge : generatedEndpoints.cluster(cluster)) { endpoints.add(builder.generatedFrom(ge).in(controller.system())); } } @@ -196,6 +272,7 @@ public class RoutingController { TreeMap<ZoneId, List<Endpoint>> endpoints = new TreeMap<>(Comparator.comparing(ZoneId::value)); for (var deployment : deployments) { EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone) + .authMethod(AuthMethod.mtls) .not().legacy(); EndpointList directEndpoints = zoneEndpoints.direct(); if (!directEndpoints.isEmpty()) { @@ -256,62 +333,47 @@ public class RoutingController { return Collections.unmodifiableList(endpointDnsNames); } - /** Returns the global and application-level endpoints for given deployment, as container endpoints */ - public Set<ContainerEndpoint> containerEndpointsOf(LockedApplication application, InstanceName instanceName, ZoneId zone) { - // Assign rotations to application - for (var deploymentInstanceSpec : application.get().deploymentSpec().instances()) { - if (deploymentInstanceSpec.concerns(Environment.prod)) { - application = controller.routing().assignRotations(application, deploymentInstanceSpec.name()); - } + /** Remove endpoints in DNS for all rotations assigned to given instance */ + public void removeRotationEndpointsFromDns(Application application, InstanceName instanceName) { + Set<Endpoint> endpointsToRemove = new LinkedHashSet<>(); + Instance instance = application.require(instanceName); + // Compute endpoints from rotations. When removing DNS records for rotation-based endpoints we cannot use the + // deployment spec, because submitting an empty deployment spec is the first step of removing an application + for (var rotation : instance.rotations()) { + var deployments = rotation.regions().stream() + .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region))) + .toList(); + endpointsToRemove.addAll(computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()), + rotation.clusterId(), deployments, readMultiDeploymentGeneratedEndpoints(application.id()))); } + endpointsToRemove.forEach(endpoint -> controller.nameServiceForwarder() + .removeRecords(Record.Type.CNAME, + RecordName.from(endpoint.dnsName()), + Priority.normal, + Optional.of(application.id()))); + } - // Add endpoints backed by a rotation, and register them in DNS if necessary - Instance instance = application.get().require(instanceName); - Set<ContainerEndpoint> containerEndpoints = new HashSet<>(); - DeploymentId deployment = new DeploymentId(instance.id(), zone); - EndpointList endpoints = declaredEndpointsOf(application.get()).targets(deployment); - EndpointList globalEndpoints = endpoints.scope(Endpoint.Scope.global); - for (var assignedRotation : instance.rotations()) { + private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { + TenantAndApplicationId owner = TenantAndApplicationId.from(prepared.deployment().applicationId()); + EndpointList globalEndpoints = prepared.endpoints().scope(Scope.global); + for (var assignedRotation : prepared.rotations()) { EndpointList rotationEndpoints = globalEndpoints.named(assignedRotation.endpointId(), Scope.global) .requiresRotation(); - // Skip rotations which do not apply to this zone - if (!assignedRotation.regions().contains(zone.region())) { + if (!assignedRotation.regions().contains(prepared.deployment().zoneId().region())) { continue; } - // Register names in DNS Rotation rotation = rotationRepository.requireRotation(assignedRotation.rotationId()); for (var endpoint : rotationEndpoints) { controller.nameServiceForwarder().createRecord( new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(rotation.name())), Priority.normal, - Optional.of(application.get().id())); - List<String> names = List.of(endpoint.dnsName(), - // Include rotation ID as a valid name of this container endpoint - // (required by global routing health checks) - assignedRotation.rotationId().asString()); - containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(), - asString(Endpoint.Scope.global), - names, - OptionalInt.empty(), - endpoint.routingMethod())); + Optional.of(owner) + ); } } - // Add endpoints not backed by a rotation (i.e. other routing methods so that the config server always knows - // about global names, even when not using rotations) - globalEndpoints.not().requiresRotation() - .groupingBy(Endpoint::cluster) - .forEach((clusterId, clusterEndpoints) -> { - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - asString(Endpoint.Scope.global), - clusterEndpoints.mapToList(Endpoint::dnsName), - OptionalInt.empty(), - RoutingMethod.exclusive)); - }); - // Add application endpoints - EndpointList applicationEndpoints = endpoints.scope(Endpoint.Scope.application); - for (var endpoint : applicationEndpoints.shared()) { // DNS for non-shared endpoints is handled by RoutingPolicies + for (var endpoint : prepared.endpoints().scope(Scope.application).shared()) { // DNS for non-shared application endpoints is handled by RoutingPolicies Set<ZoneId> targetZones = endpoint.targets().stream() .map(t -> t.deployment().zoneId()) .collect(Collectors.toUnmodifiableSet()); @@ -324,79 +386,31 @@ public class RoutingController { controller.nameServiceForwarder().createRecord( new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(vipHostname)), Priority.normal, - Optional.of(application.get().id())); - } - Map<ClusterSpec.Id, EndpointList> applicationEndpointsByCluster = applicationEndpoints.groupingBy(Endpoint::cluster); - for (var kv : applicationEndpointsByCluster.entrySet()) { - ClusterSpec.Id clusterId = kv.getKey(); - EndpointList clusterEndpoints = kv.getValue(); - for (var endpoint : clusterEndpoints) { - Optional<Endpoint.Target> matchingTarget = endpoint.targets().stream() - .filter(t -> t.routesTo(deployment)) - .findFirst(); - if (matchingTarget.isEmpty()) throw new IllegalStateException("No target found routing to " + deployment + " in " + endpoint); - containerEndpoints.add(new ContainerEndpoint(clusterId.value(), - asString(Endpoint.Scope.application), - List.of(endpoint.dnsName()), - OptionalInt.of(matchingTarget.get().weight()), - endpoint.routingMethod())); - } + Optional.of(owner)); } - return Collections.unmodifiableSet(containerEndpoints); - } - - /** Remove endpoints in DNS for all rotations assigned to given instance */ - public void removeEndpointsInDns(Application application, InstanceName instanceName) { - Set<Endpoint> endpointsToRemove = new LinkedHashSet<>(); - Instance instance = application.require(instanceName); - // Compute endpoints from rotations. When removing DNS records for rotation-based endpoints we cannot use the - // deployment spec, because submitting an empty deployment spec is the first step of removing an application - for (var rotation : instance.rotations()) { - var deployments = rotation.regions().stream() - .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region))) - .toList(); - endpointsToRemove.addAll(computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()), - rotation.clusterId(), deployments, readGeneratedEndpoints(application))); - } - endpointsToRemove.forEach(endpoint -> controller.nameServiceForwarder() - .removeRecords(Record.Type.CNAME, - RecordName.from(endpoint.dnsName()), - Priority.normal, - Optional.of(application.id()))); } /** Generate endpoints for all authentication methods, using given application part */ - public List<GeneratedEndpoint> generateEndpoints(String applicationPart, ApplicationId instance) { + private List<GeneratedEndpoint> generateEndpoints(String applicationPart, ApplicationId instance, boolean token) { if (!randomizedEndpointsEnabled(instance)) { return List.of(); } - return generateEndpoints(applicationPart); - } - - - private List<GeneratedEndpoint> generateEndpoints(String applicationPart) { return Arrays.stream(AuthMethod.values()) + .filter(method -> method != AuthMethod.token || token) .map(method -> new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), applicationPart, method)) .toList(); } - /** This is only suitable for use in declared endpoints, which ignore the randomly generated cluster part */ - private List<GeneratedEndpoint> readGeneratedEndpoints(Application application) { - boolean includeTokenEndpoint = application.productionInstances().values().stream() - .map(Instance::id) - .anyMatch(this::tokenEndpointEnabled); - Optional<String> randomizedId = controller.curator().readAssignedCertificate(application.id(), Optional.empty()) - .map(AssignedCertificate::certificate) - .flatMap(EndpointCertificate::randomizedId); - if (randomizedId.isEmpty()) { - return List.of(); + /** Returns generated endpoint suitable for use in endpoints whose scope is {@link Scope#multiDeployment()} */ + private GeneratedEndpoints readMultiDeploymentGeneratedEndpoints(TenantAndApplicationId application) { + Map<ClusterSpec.Id, List<GeneratedEndpoint>> endpoints = new HashMap<>(); + for (var policy : policies().read(application)) { + // The cluster part is not used in this context because multi-deployment endpoints have a user-controlled name + endpoints.putIfAbsent(policy.id().cluster(), policy.generatedEndpoints().stream().toList()); } - return generateEndpoints(randomizedId.get()).stream().filter(endpoint -> switch (endpoint.authMethod()) { - case token -> includeTokenEndpoint; - case mtls -> true; - }).toList(); + return new GeneratedEndpoints(endpoints); } /** @@ -436,7 +450,7 @@ public class RoutingController { } /** Compute global endpoints for given routing ID, application and deployments */ - private List<Endpoint> computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, List<GeneratedEndpoint> generatedEndpoints) { + private List<Endpoint> computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, GeneratedEndpoints generatedEndpoints) { var endpoints = new ArrayList<Endpoint>(); var directMethods = 0; var availableRoutingMethods = routingMethodsOfAll(deployments); @@ -450,14 +464,15 @@ public class RoutingController { .on(Port.fromRoutingMethod(method)) .routingMethod(method); endpoints.add(builder.in(controller.system())); - for (var ge : generatedEndpoints) { + for (var ge : generatedEndpoints.cluster(cluster)) { endpoints.add(builder.generatedFrom(ge).in(controller.system())); } } return endpoints; } - public boolean tokenEndpointEnabled(ApplicationId instance) { + + private boolean tokenEndpointEnabled(ApplicationId instance) { return createTokenEndpoint.with(FetchVector.Dimension.APPLICATION_ID, instance.serializedForm()).value(); } @@ -473,13 +488,5 @@ public class RoutingController { return 'v' + base32 + Endpoint.internalDnsSuffix(system); } - private static String asString(Endpoint.Scope scope) { - return switch (scope) { - case application -> "application"; - case global -> "global"; - case weighted -> "weighted"; - case zone -> "zone"; - }; - } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index 5f7f59e6cdc..010bc023dad 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -166,7 +166,7 @@ public class Endpoint { @Override public String toString() { - return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s]", url, scope, legacy, routingMethod, authMethod); + return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s, name=%s]", url, scope, legacy, routingMethod, authMethod, name()); } private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index dcc3e229f92..310a78e45f0 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import com.yahoo.collections.AbstractFilteringList; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.InstanceName; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import java.util.Collection; @@ -94,10 +95,19 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(endpoint -> endpoint.routingMethod().isShared()); } + /** Returns the subset of endpoints supporting given authentication method */ + public EndpointList authMethod(AuthMethod authMethod) { + return matching(endpoint -> endpoint.authMethod() == authMethod); + } + public static EndpointList copyOf(Collection<Endpoint> endpoints) { return new EndpointList(endpoints, false); } + public static EndpointList of(Endpoint ...endpoint) { + return copyOf(List.of(endpoint)); + } + @Override public String toString() { return asList().toString(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java index 3ec79b03ee8..3ec7f120726 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java @@ -22,8 +22,11 @@ import com.yahoo.vespa.archive.ArchiveStreamReader; import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile; import com.yahoo.vespa.archive.ArchiveStreamReader.Options; import com.yahoo.vespa.hosted.controller.Application; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder; import com.yahoo.yolean.Exceptions; +import org.w3c.dom.Document; + import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStreamReader; @@ -136,6 +139,25 @@ public class ApplicationPackage { */ public ValidationOverrides validationOverrides() { return validationOverrides; } + /** Returns a basic variant of services.xml contained in this package, pre-processed according to given deployment and tags */ + public BasicServicesXml services(DeploymentId deployment, Tags tags) { + FileWrapper servicesXml = files.wrapper().wrap(Paths.get(servicesFile)); + if (!servicesXml.exists()) return BasicServicesXml.empty; + try { + Document document = new XmlPreProcessor(files.wrapper().wrap(Paths.get("./")), + new InputStreamReader(new ByteArrayInputStream(servicesXml.content()), UTF_8), + deployment.applicationId().instance(), + deployment.zoneId().environment(), + deployment.zoneId().region(), + tags).run(); + return BasicServicesXml.parse(document); + } catch (IllegalArgumentException e) { + throw e; + } catch (Exception e) { + throw new IllegalArgumentException(e); + } + } + /** Returns the platform version which package was compiled against, if known. */ public Optional<Version> compileVersion() { return compileVersion; } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 9394a1fcbe2..a0e8b1c5610 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -333,7 +333,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { List<EndpointTarget> targets = new ArrayList<>(); out: for (var app : applications) { - Optional<Endpoint> declaredEndpoint = controller.routing().declaredEndpointsOf(app).dnsName(endpoint); + Optional<Endpoint> declaredEndpoint = controller.routing().readDeclaredEndpointsOf(app).dnsName(endpoint); if (declaredEndpoint.isPresent()) { for (var target : declaredEndpoint.get().targets()) { targets.add(new EndpointTarget(target.deployment(), declaredEndpoint.get().cluster())); @@ -2061,7 +2061,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { if (!legacyEndpoints) { zoneEndpoints = zoneEndpoints.not().legacy().direct(); } - EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application).targets(deploymentId); + EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application).targets(deploymentId); if (!legacyEndpoints) { declaredEndpoints = declaredEndpoints.not().legacy().direct(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java index 232f25f5674..bc83eeb73c1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java @@ -253,7 +253,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler { var instances = instanceId == null ? application.instances().values() : List.of(application.require(instanceId.instance())); - EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application); + EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application); for (var instance : instances) { var zones = zoneId == null ? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)).toList() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java new file mode 100644 index 00000000000..3adbb43a7b5 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java @@ -0,0 +1,32 @@ +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * This represents endpoints generated by the controller for a deployment, grouped by their assigned cluster. + * + * @author mpolden + */ +public record GeneratedEndpoints(Map<ClusterSpec.Id, List<GeneratedEndpoint>> endpoints) { + + public static final GeneratedEndpoints empty = new GeneratedEndpoints(Map.of()); + + public GeneratedEndpoints(Map<ClusterSpec.Id, List<GeneratedEndpoint>> endpoints) { + this.endpoints = Map.copyOf(Objects.requireNonNull(endpoints)); + endpoints.forEach((cluster, generatedEndpoints) -> { + if (generatedEndpoints.stream().distinct().count() != generatedEndpoints.size()) { + throw new IllegalStateException("Endpoints for " + cluster + " must be distinct, got " + generatedEndpoints); + } + }); + } + + public List<GeneratedEndpoint> cluster(ClusterSpec.Id cluster) { + return endpoints.getOrDefault(cluster, List.of()); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java new file mode 100644 index 00000000000..c67d88fa81f --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java @@ -0,0 +1,121 @@ +package com.yahoo.vespa.hosted.controller.routing; + +import com.yahoo.config.provision.ClusterSpec; +import com.yahoo.config.provision.zone.AuthMethod; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; +import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; +import com.yahoo.vespa.hosted.controller.application.AssignedRotation; +import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +/** + * This represents the endpoints, and associated resources, that have been prepared for a deployment. + * + * @author mpolden + */ +public record PreparedEndpoints(DeploymentId deployment, + EndpointList endpoints, + List<AssignedRotation> rotations, + Optional<EndpointCertificate> certificate) { + + public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, Optional<EndpointCertificate> certificate) { + this.deployment = Objects.requireNonNull(deployment); + this.endpoints = Objects.requireNonNull(endpoints); + this.rotations = List.copyOf(Objects.requireNonNull(rotations)); + this.certificate = Objects.requireNonNull(certificate); + } + + /** Returns the endpoints generated by this prepare */ + public GeneratedEndpoints generatedEndpoints() { + Map<ClusterSpec.Id, List<GeneratedEndpoint>> generated = new HashMap<>(); + for (var endpoint : endpoints.generated()) { + List<GeneratedEndpoint> clusterGenerated = generated.computeIfAbsent(endpoint.cluster(), (k) -> new ArrayList<>()); + if (!clusterGenerated.contains(endpoint.generated().get())) { + clusterGenerated.add(endpoint.generated().get()); + } + } + return new GeneratedEndpoints(generated); + } + + /** Returns the endpoints contained in this as {@link com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint} */ + public Set<ContainerEndpoint> containerEndpoints() { + Map<EndpointId, AssignedRotation> rotationsByEndpointId = rotations.stream() + .collect(Collectors.toMap(AssignedRotation::endpointId, + Function.identity())); + Set<ContainerEndpoint> containerEndpoints = new HashSet<>(); + endpoints.scope(Endpoint.Scope.zone).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> { + containerEndpoints.add(new ContainerEndpoint(clusterId.value(), + asString(Endpoint.Scope.zone), + clusterEndpoints.mapToList(Endpoint::dnsName), + OptionalInt.empty(), + clusterEndpoints.first().get().routingMethod(), + authMethodsByDnsName(clusterEndpoints))); + }); + endpoints.scope(Endpoint.Scope.global).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> { + for (var endpoint : clusterEndpoints) { + List<String> names = new ArrayList<>(2); + names.add(endpoint.dnsName()); + if (endpoint.requiresRotation()) { + EndpointId endpointId = EndpointId.of(endpoint.name()); + AssignedRotation rotation = rotationsByEndpointId.get(endpointId); + if (rotation == null) { + throw new IllegalArgumentException(endpoint + " requires a rotation, but no rotation has been assigned to " + endpointId); + } + // Include the rotation ID as a valid name of this container endpoint + // (required by global routing health checks) + names.add(rotation.rotationId().asString()); + } + containerEndpoints.add(new ContainerEndpoint(clusterId.value(), + asString(Endpoint.Scope.global), + names, + OptionalInt.empty(), + endpoint.routingMethod(), + authMethodsByDnsName(EndpointList.of(endpoint)))); + } + }); + endpoints.scope(Endpoint.Scope.application).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> { + for (var endpoint : clusterEndpoints) { + Optional<Endpoint.Target> matchingTarget = endpoint.targets().stream() + .filter(t -> t.routesTo(deployment)) + .findFirst(); + if (matchingTarget.isEmpty()) throw new IllegalStateException("No target found routing to " + deployment + " in " + endpoint); + containerEndpoints.add(new ContainerEndpoint(clusterId.value(), + asString(Endpoint.Scope.application), + List.of(endpoint.dnsName()), + OptionalInt.of(matchingTarget.get().weight()), + endpoint.routingMethod(), + authMethodsByDnsName(EndpointList.of(endpoint)))); + } + }); + return containerEndpoints; + } + + private static Map<String, AuthMethod> authMethodsByDnsName(EndpointList endpoints) { + return endpoints.asList().stream().collect(Collectors.toMap(Endpoint::dnsName, Endpoint::authMethod)); + } + + private static String asString(Endpoint.Scope scope) { + return switch (scope) { + case application -> "application"; + case global -> "global"; + case weighted -> "weighted"; + case zone -> "zone"; + }; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index c8c3d057ee3..eb881519589 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.routing; import ai.vespa.http.DomainName; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.transaction.Mutex; @@ -43,6 +44,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.function.Function; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -107,7 +109,7 @@ public class RoutingPolicies { * Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if * routing configuration affecting given deployment has changed. */ - public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) { + public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, GeneratedEndpoints generatedEndpoints) { ApplicationId instance = deployment.applicationId(); List<LoadBalancer> loadBalancers = controller.serviceRegistry().configServer() .getLoadBalancers(instance, deployment.zoneId()); @@ -243,14 +245,25 @@ public class RoutingPolicies { for (var policy : policies) { if (policy.dnsZone().isEmpty() && policy.canonicalName().isPresent()) continue; if (controller.zoneRegistry().routingMethod(policy.id().zone()) != RoutingMethod.exclusive) continue; - Endpoint endpoint = policy.regionEndpointIn(controller.system(), RoutingMethod.exclusive, parent.generated()); var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone()); - long weight = 1; - if (isConfiguredOut(zonePolicy, policy)) { - weight = 0; // A record with 0 weight will not receive traffic. If all records within a group have 0 - // weight, traffic is routed to all records with equal probability. + // A record with 0 weight will not receive traffic. If all records within a group have 0 + // weight, traffic is routed to all records with equal probability + long weight = isConfiguredOut(zonePolicy, policy) ? 0 : 1; + boolean generated = parent.generated().isPresent(); + EndpointList weightedEndpoints = controller.routing() + .endpointsOf(policy.id().deployment(), + policy.id().cluster(), + parent.generated().stream().toList()) + .scope(Endpoint.Scope.weighted); + if (generated) { + weightedEndpoints = weightedEndpoints.generated(); + } else { + weightedEndpoints = weightedEndpoints.not().generated(); } - + if (weightedEndpoints.size() != 1) { + throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent); + } + Endpoint endpoint = weightedEndpoints.first().get(); RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint( new LatencyAliasTarget(DomainName.of(endpoint.dnsName()), policy.dnsZone().get(), policy.id().zone()))); @@ -282,7 +295,7 @@ public class RoutingPolicies { Map<Endpoint, Set<Target>> inactiveTargetsByEndpoint = new LinkedHashMap<>(); for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) { RoutingId routingId = routeEntry.getKey(); - EndpointList endpoints = controller.routing().declaredEndpointsOf(application) + EndpointList endpoints = controller.routing().readDeclaredEndpointsOf(application) .named(routingId.endpointId(), Endpoint.Scope.application); for (Endpoint endpoint : endpoints) { for (var policy : routeEntry.getValue()) { @@ -355,22 +368,23 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, List<GeneratedEndpoint> generatedEndpoints, @SuppressWarnings("unused") Mutex lock) { + private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, GeneratedEndpoints generatedEndpoints, @SuppressWarnings("unused") Mutex lock) { Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(applicationPolicies.instance(allocation.deployment.applicationId()).asMap()); for (LoadBalancer loadBalancer : allocation.loadBalancers) { if (loadBalancer.hostname().isEmpty() && loadBalancer.ipAddress().isEmpty()) continue; var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); var existingPolicy = policies.get(policyId); var dnsZone = loadBalancer.ipAddress().isPresent() ? Optional.of("ignored") : loadBalancer.dnsZone(); + var clusterGeneratedEndpoints = generatedEndpoints.cluster(loadBalancer.cluster()); var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.ipAddress(), dnsZone, allocation.instanceEndpointsOf(loadBalancer), allocation.applicationEndpointsOf(loadBalancer), RoutingStatus.DEFAULT, loadBalancer.isPublic(), - generatedEndpoints); - boolean addingGeneratedEndpoints = !generatedEndpoints.isEmpty() && (existingPolicy == null || existingPolicy.generatedEndpoints().isEmpty()); + clusterGeneratedEndpoints); + boolean addingGeneratedEndpoints = !clusterGeneratedEndpoints.isEmpty() && (existingPolicy == null || existingPolicy.generatedEndpoints().isEmpty()); if (addingGeneratedEndpoints) { - generatedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies)); + clusterGeneratedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies)); } if (existingPolicy != null) { newPolicy = newPolicy.with(existingPolicy.routingStatus()); // Always preserve routing status @@ -386,11 +400,17 @@ public class RoutingPolicies { return updated; } + private static Map<AuthMethod, GeneratedEndpoint> asMap(List<GeneratedEndpoint> generatedEndpoints) { + return generatedEndpoints.stream().collect(Collectors.toMap(GeneratedEndpoint::authMethod, Function.identity())); + } + /** Update zone DNS record for given policy */ private void updateZoneDnsOf(RoutingPolicy policy, LoadBalancer loadBalancer, DeploymentId deploymentId) { - RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deploymentId.zoneId()); - boolean addTokenEndpoint = controller.routing().tokenEndpointEnabled(deploymentId.applicationId()); - for (var endpoint : policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)) { + EndpointList zoneEndpoints = controller.routing().endpointsOf(deploymentId, + policy.id().cluster(), + policy.generatedEndpoints()) + .scope(Endpoint.Scope.zone); + for (var endpoint : zoneEndpoints) { RecordName name = RecordName.from(endpoint.dnsName()); Record record = policy.canonicalName().isPresent() ? new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) : @@ -464,14 +484,16 @@ public class RoutingPolicies { * @return the updated policies */ private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { - RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(allocation.deployment.zoneId()); - boolean addTokenEndpoint = controller.routing().tokenEndpointEnabled(allocation.deployment.applicationId()); Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap()); Set<RoutingPolicyId> activeIds = allocation.asPolicyIds(); RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment) .not().matching(policy -> activeIds.contains(policy.id())); for (var policy : removable) { - for (var endpoint : policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)) { + EndpointList zoneEndpoints = controller.routing().endpointsOf(allocation.deployment, + policy.id().cluster(), + policy.generatedEndpoints()) + .scope(Endpoint.Scope.zone); + for (var endpoint : zoneEndpoints) { Record.Type type = policy.canonicalName().isPresent() ? Record.Type.CNAME : Record.Type.A; nameServiceForwarder(endpoint).removeRecords(type, RecordName.from(endpoint.dnsName()), diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index f8c3bd7cf7c..2363524e306 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -3,16 +3,10 @@ package com.yahoo.vespa.hosted.controller.routing; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSortedSet; -import com.yahoo.config.provision.SystemName; -import com.yahoo.config.provision.zone.AuthMethod; -import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -117,36 +111,6 @@ public record RoutingPolicy(RoutingPolicyId id, return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints); } - /** Returns the zone endpoints of this */ - public List<Endpoint> zoneEndpointsIn(SystemName system, RoutingMethod routingMethod, boolean includeTokenEndpoint) { - DeploymentId deployment = new DeploymentId(id.owner(), id.zone()); - Endpoint.EndpointBuilder builder = endpoint(routingMethod).target(id.cluster(), deployment); - Endpoint zoneEndpoint = builder.in(system); - List<Endpoint> endpoints = new ArrayList<>(); - endpoints.add(zoneEndpoint); - if (includeTokenEndpoint) { - Endpoint tokenEndpoint = builder.authMethod(AuthMethod.token).in(system); - endpoints.add(tokenEndpoint); - } - for (var generatedEndpoint : generatedEndpoints) { - boolean include = switch (generatedEndpoint.authMethod()) { - case token -> includeTokenEndpoint; - case mtls -> true; - }; - if (include) { - endpoints.add(builder.generatedFrom(generatedEndpoint).in(system)); - } - } - return endpoints; - } - - /** Returns the region endpoint of this */ - public Endpoint regionEndpointIn(SystemName system, RoutingMethod routingMethod, Optional<GeneratedEndpoint> generated) { - Endpoint.EndpointBuilder builder = endpoint(routingMethod).targetRegion(id.cluster(), id.zone()); - generated.ifPresent(builder::generatedFrom); - return builder.in(system); - } - @Override public boolean equals(Object o) { if (this == o) return true; @@ -160,10 +124,4 @@ public record RoutingPolicy(RoutingPolicyId id, return Objects.hash(id); } - private Endpoint.EndpointBuilder endpoint(RoutingMethod routingMethod) { - return Endpoint.of(id.owner()) - .on(Port.fromRoutingMethod(routingMethod)) - .routingMethod(routingMethod); - } - } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index 2e11a156dce..64a969a9c9d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -8,10 +8,12 @@ import com.yahoo.vespa.hosted.controller.LockedApplication; import com.yahoo.vespa.hosted.controller.RoutingController; import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; -import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; +import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; +import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpoints; +import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; @@ -20,22 +22,21 @@ import java.time.Clock; import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** - * A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment. + * A deployment routing context. This extends {@link RoutingContext} to support configuration of routing for a deployment. * * @author mpolden */ public abstract class DeploymentRoutingContext implements RoutingContext { final DeploymentId deployment; - final RoutingController controller; + final RoutingController routing; final RoutingMethod method; - public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController controller) { + public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController routing) { this.deployment = Objects.requireNonNull(deployment); - this.controller = Objects.requireNonNull(controller); + this.routing = Objects.requireNonNull(routing); this.method = Objects.requireNonNull(method); } @@ -44,13 +45,13 @@ public abstract class DeploymentRoutingContext implements RoutingContext { * * @return the container endpoints relevant for this deployment, as declared in deployment spec */ - public final Set<ContainerEndpoint> prepare(LockedApplication application) { - return controller.containerEndpointsOf(application, deployment.applicationId().instance(), deployment.zoneId()); + public final PreparedEndpoints prepare(BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) { + return routing.prepare(deployment, services, certificate, application); } - /** Configure routing for the deployment in this context, using given deployment spec */ - public final void configure(DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) { - controller.policies().refresh(deployment, deploymentSpec, generatedEndpoints); + /** Finalize routing configuration for the deployment in this context, using given deployment spec */ + public final void activate(DeploymentSpec deploymentSpec, GeneratedEndpoints generatedEndpoints) { + routing.policies().refresh(deployment, deploymentSpec, generatedEndpoints); } /** Routing method of this context */ @@ -61,7 +62,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { /** Read the routing policy for given cluster in this deployment */ public final Optional<RoutingPolicy> routingPolicy(ClusterSpec.Id cluster) { RoutingPolicyId id = new RoutingPolicyId(deployment.applicationId(), cluster, deployment.zoneId()); - return controller.policies().read(deployment).of(id); + return routing.policies().read(deployment).of(id); } /** Extension of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#sharedLayer4} routing */ @@ -110,13 +111,13 @@ public abstract class DeploymentRoutingContext implements RoutingContext { } private List<String> upstreamNames() { - List<String> upstreamNames = controller.readEndpointsOf(deployment) - .scope(Endpoint.Scope.zone) - .shared() - .asList().stream() - .map(endpoint -> endpoint.upstreamName(deployment)) - .distinct() - .toList(); + List<String> upstreamNames = routing.readEndpointsOf(deployment) + .scope(Endpoint.Scope.zone) + .shared() + .asList().stream() + .map(endpoint -> endpoint.upstreamName(deployment)) + .distinct() + .toList(); if (upstreamNames.isEmpty()) { throw new IllegalArgumentException("No upstream names found for " + deployment); } @@ -137,17 +138,17 @@ public abstract class DeploymentRoutingContext implements RoutingContext { @Override public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) { - controller.policies().setRoutingStatus(deployment, value, agent); + routing.policies().setRoutingStatus(deployment, value, agent); } @Override public RoutingStatus routingStatus() { // Status for a deployment applies to all clusters within the deployment, so we use the status from the // first matching policy here - return controller.policies().read(deployment) - .first() - .map(RoutingPolicy::routingStatus) - .orElse(RoutingStatus.DEFAULT); + return routing.policies().read(deployment) + .first() + .map(RoutingPolicy::routingStatus) + .orElse(RoutingStatus.DEFAULT); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index c46a28c4567..1ac811f0b4f 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -16,11 +16,13 @@ import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.TenantName; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.path.Path; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData; +import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentEndpoints; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock; import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota; @@ -332,7 +334,8 @@ public class ControllerTest { List.of("beta.app1.tenant1.global.vespa.oath.cloud", "rotation-id-01"), OptionalInt.empty(), - RoutingMethod.sharedLayer4)); + RoutingMethod.sharedLayer4, + Map.of("beta.app1.tenant1.global.vespa.oath.cloud", AuthMethod.mtls))); for (Deployment deployment : betaDeployments) { assertEquals(containerEndpoints, @@ -350,7 +353,8 @@ public class ControllerTest { List.of("app1.tenant1.global.vespa.oath.cloud", "rotation-id-02"), OptionalInt.empty(), - RoutingMethod.sharedLayer4)); + RoutingMethod.sharedLayer4, + Map.of("app1.tenant1.global.vespa.oath.cloud", AuthMethod.mtls))); for (Deployment deployment : defaultDeployments) { assertEquals(containerEndpoints, tester.configServer().containerEndpoints().get(defaultContext.deploymentIdIn(deployment.zone()))); @@ -740,10 +744,14 @@ public class ControllerTest { ); deploymentEndpoints.forEach((deployment, endpoints) -> { Set<ContainerEndpoint> expected = endpoints.entrySet().stream() - .map(kv -> new ContainerEndpoint("default", "application", + .map(kv -> { + Map<String, AuthMethod> authMethods = kv.getKey().stream().collect(Collectors.toMap(Function.identity(), (v) -> AuthMethod.mtls)); + return new ContainerEndpoint("default", "application", kv.getKey(), OptionalInt.of(kv.getValue()), - tester.controller().zoneRegistry().routingMethod(deployment.zoneId()))) + tester.controller().zoneRegistry().routingMethod(deployment.zoneId()), + authMethods); + }) .collect(Collectors.toSet()); assertEquals(expected, tester.configServer().containerEndpoints().get(deployment), @@ -790,7 +798,7 @@ public class ControllerTest { RecordName.from("e.app1.tenant1.a.vespa.oath.cloud"), RecordData.from("vip.prod.us-east-3.")))), new TreeSet<>(records)); - List<String> endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application()) + List<String> endpointDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.application()) .scope(Endpoint.Scope.application) .sortedBy(comparing(Endpoint::dnsName)) .mapToList(Endpoint::dnsName); @@ -1536,8 +1544,8 @@ public class ControllerTest { DeploymentContext context = tester.newDeploymentContext(); DeploymentId deployment = context.deploymentIdIn(ZoneId.from("prod", "us-west-1")); DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"), - Set.of(), Optional::empty, Optional.empty(), Optional.empty(), - Quota::unlimited, List.of(), List.of(), Optional::empty, List.of(),false); + () -> DeploymentEndpoints.none, Optional.empty(), Optional.empty(), + Quota::unlimited, List.of(), List.of(), Optional::empty, List.of(), false); tester.configServer().deploy(deploymentData); assertTrue(tester.configServer().application(deployment.applicationId(), deployment.zoneId()).isPresent()); tester.controller().applications().deactivate(deployment.applicationId(), deployment.zoneId()); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java index fb3026e1d80..f417e3d52fb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java @@ -8,6 +8,7 @@ import com.yahoo.config.provision.AthenzService; import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; @@ -56,6 +57,7 @@ public class ApplicationPackageBuilder { "/>\n</notifications>\n").setEmptyValue(""); private final StringBuilder endpointsBody = new StringBuilder(); private final StringBuilder applicationEndpointsBody = new StringBuilder(); + private final StringBuilder servicesBody = new StringBuilder(); private final List<X509Certificate> trustedCertificates = new ArrayList<>(); private final Map<Environment, Map<String, String>> nonProductionEnvironments = new LinkedHashMap<>(); @@ -112,6 +114,28 @@ public class ApplicationPackageBuilder { return this; } + public ApplicationPackageBuilder container(String id, AuthMethod... authMethod) { + servicesBody.append(" <container id='") + .append(id) + .append("'>\n") + .append(" <clients>\n"); + for (int i = 0; i < authMethod.length; i++) { + AuthMethod m = authMethod[i]; + servicesBody.append(" <client id='") + .append("client-").append(m.name()).append("-").append(i) + .append("'>\n"); + if (m == AuthMethod.token) { + servicesBody.append(" <token id='") + .append(m.name()).append("-").append(i) + .append("'/>\n"); + } + servicesBody.append(" </client>\n"); + } + servicesBody.append(" </clients>\n") + .append(" </container>\n"); + return this; + } + public ApplicationPackageBuilder applicationEndpoint(String id, String containerId, String region, Map<InstanceName, Integer> instanceWeights) { return applicationEndpoint(id, containerId, Map.of(region, instanceWeights)); @@ -350,6 +374,10 @@ public class ApplicationPackageBuilder { return searchDefinition.getBytes(UTF_8); } + private byte[] services() { + return ("<services version='1.0'>\n" + servicesBody + "</services>\n").getBytes(UTF_8); + } + private static byte[] buildMeta(Version compileVersion) { return compileVersion == null ? new byte[0] : ("{\"compileVersion\":\"" + compileVersion.toFullString() + @@ -362,6 +390,7 @@ public class ApplicationPackageBuilder { try (ZipOutputStream out = new ZipOutputStream(zip)) { out.setLevel(Deflater.NO_COMPRESSION); // This is for testing purposes so we skip compression for performance writeZipEntry(out, "deployment.xml", deploymentSpec()); + writeZipEntry(out, "services.xml", services()); writeZipEntry(out, "validation-overrides.xml", validationOverrides()); writeZipEntry(out, "schemas/test.sd", searchDefinition()); writeZipEntry(out, "build-meta.json", buildMeta(compileVersion)); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java index 0862496275a..a03583c4a59 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java @@ -409,12 +409,12 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer applications.put(id, new Application(id.applicationId(), lastPrepareVersion, appPackage)); ClusterSpec.Id cluster = ClusterSpec.Id.from("default"); - deployment.endpointCertificate(); // Supplier with side effects >_< + deployment.endpoints(); // Supplier with side effects >_< if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty()) provision(id.zoneId(), id.applicationId(), cluster); - this.containerEndpoints.put(id, deployment.containerEndpoints()); + this.containerEndpoints.put(id, deployment.endpoints().get().endpoints()); deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account)); if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 46ec42cab8f..1dfaf2109c7 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -14,6 +14,7 @@ import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; +import com.yahoo.config.provision.zone.AuthMethod; import com.yahoo.config.provision.zone.RoutingMethod; import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.vespa.flags.Flags; @@ -41,6 +42,7 @@ import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue; import com.yahoo.vespa.hosted.controller.dns.RemoveRecords; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.Timeout; @@ -334,6 +336,10 @@ public class RoutingPoliciesTest { var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); // Deploy application + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .container("c0", AuthMethod.mtls, AuthMethod.token) + .build(); tester.provisionLoadBalancers(1, context1.instanceId(), false, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); @@ -358,12 +364,23 @@ public class RoutingPoliciesTest { // Ordinary endpoints are not created in DNS assertEquals(List.of(), tester.recordNames()); assertEquals(2, tester.policiesOf(context.instanceId()).size()); - // Generated endpoints are created in DNS + } + + @Test + @Disabled // TODO(mpolden): Enable this test when we start creating generated endpoints for shared routing + void zone_routing_policies_with_shared_routing_and_generated_endpoint() { + var tester = new RoutingPoliciesTester(new DeploymentTester(), false); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2); tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .container("c0", AuthMethod.mtls, AuthMethod.token) + .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - assertEquals(List.of("b22ab332.cafed00d.z.vespa.oath.cloud", - "d71005bf.cafed00d.z.vespa.oath.cloud"), + assertEquals(List.of("c0a25b7c.cafed00d.z.vespa.oath.cloud", + "dc5e383c.cafed00d.z.vespa.oath.cloud"), tester.recordNames()); } @@ -757,7 +774,7 @@ public class RoutingPoliciesTest { tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); } catch (IllegalArgumentException e) { - assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive, authMethod=mtls]", e.getMessage()); + assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive, authMethod=mtls, name=r0]", e.getMessage()); } context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); @@ -942,7 +959,7 @@ public class RoutingPoliciesTest { tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive, authMethod=mtls]", + assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive, authMethod=mtls, name=a0]", e.getMessage()); } @@ -993,14 +1010,17 @@ public class RoutingPoliciesTest { var tester = new RoutingPoliciesTester(SystemName.Public); var context = tester.newDeploymentContext("tenant1", "app1", "default"); tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + tester.enableTokenEndpoint(true); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application - int clustersPerZone = 1; + int clustersPerZone = 2; var zone1 = ZoneId.from("prod", "aws-us-east-1c"); var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) .region(zone2.region()) + .container("c0", AuthMethod.mtls) + .container("c1", AuthMethod.mtls, AuthMethod.token) .endpoint("foo", "c0") .applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1))) .build(); @@ -1011,6 +1031,8 @@ public class RoutingPoliciesTest { List<String> expectedRecords = List.of( // save me, jebus! "b22ab332.cafed00d.z.vespa-app.cloud", + "b7e79800.cafed00d.z.vespa-app.cloud", + "b8ee0967.cafed00d.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", "bar.cafed00d.a.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", @@ -1019,26 +1041,42 @@ public class RoutingPoliciesTest { "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c0.cafed00d.aws-eu-west-1.w.vespa-app.cloud", "c0.cafed00d.aws-us-east-1.w.vespa-app.cloud", - "dd0971b4.cafed00d.z.vespa-app.cloud", + "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", + "c60d3149.cafed00d.z.vespa-app.cloud", + "cbff1506.cafed00d.z.vespa-app.cloud", + "d151139b.cafed00d.z.vespa-app.cloud", "foo.app1.tenant1.g.vespa-app.cloud", - "foo.cafed00d.g.vespa-app.cloud" + "foo.cafed00d.g.vespa-app.cloud", + "token-c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "token-c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud" ); assertEquals(expectedRecords, tester.recordNames()); - assertEquals(2, tester.policiesOf(context.instanceId()).size()); + assertEquals(4, tester.policiesOf(context.instanceId()).size()); + ClusterSpec.Id cluster0 = ClusterSpec.Id.from("c0"); + ClusterSpec.Id cluster1 = ClusterSpec.Id.from("c1"); for (var zone : List.of(zone1, zone2)) { - EndpointList endpoints = tester.controllerTester().controller().routing().readEndpointsOf(context.deploymentIdIn(zone)).scope(Endpoint.Scope.zone); - assertEquals(1, endpoints.generated().size()); + EndpointList generated = tester.controllerTester().controller().routing() + .readEndpointsOf(context.deploymentIdIn(zone)) + .scope(Endpoint.Scope.zone) + .generated(); + assertEquals(1, generated.cluster(cluster0).size()); + assertEquals(0, generated.cluster(cluster0).authMethod(AuthMethod.token).size()); + assertEquals(2, generated.cluster(cluster1).size()); + assertEquals(1, generated.cluster(cluster1).authMethod(AuthMethod.token).size()); } + // Ordinary endpoints point to expected targets - tester.assertTargets(context.instanceId(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0, + tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0, Map.of(zone1, 1L, zone2, 1L)); - tester.assertTargets(context.application().id(), EndpointId.of("bar"), ClusterSpec.Id.from("c0"), 0, + tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0, Map.of(context.deploymentIdIn(zone1), 1)); + // Generated endpoints point to expected targets - tester.assertTargets(context.instanceId(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0, + tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0, Map.of(zone1, 1L, zone2, 1L), true); - tester.assertTargets(context.application().id(), EndpointId.of("bar"), ClusterSpec.Id.from("c0"), 0, + tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0, Map.of(context.deploymentIdIn(zone1), 1), true); |