summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMorten Tokle <mortent@yahooinc.com>2023-08-31 10:39:11 +0200
committerGitHub <noreply@github.com>2023-08-31 10:39:11 +0200
commite2042808ce13c536ff1d7e9a6cc708849689cbae (patch)
tree62f026bb9ada8c39fa20cd649f27008fc6d2f1e3
parent932134c711b00e41012aac03ab68fb449b279092 (diff)
parentab329a2949cea94de8e6a7003adb00d987e9bb9f (diff)
Merge pull request #28276 from vespa-engine/mpolden/precompute-endpoints
Precompute zone endpoints on deployment
-rw-r--r--config-provisioning/src/main/java/com/yahoo/config/provision/zone/AuthMethod.java14
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java20
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentEndpoints.java25
-rw-r--r--controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java74
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java92
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java288
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java9
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java22
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java92
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java4
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java2
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java32
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java121
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java58
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java41
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java51
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java22
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java5
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java54
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java29
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java2
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java4
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java68
27 files changed, 777 insertions, 381 deletions
diff --git a/config-provisioning/src/main/java/com/yahoo/config/provision/zone/AuthMethod.java b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/AuthMethod.java
new file mode 100644
index 00000000000..88b8a05c4c6
--- /dev/null
+++ b/config-provisioning/src/main/java/com/yahoo/config/provision/zone/AuthMethod.java
@@ -0,0 +1,14 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.config.provision.zone;
+
+/**
+ * An endpoint's authentication method.
+ *
+ * @author mpolden
+ */
+public enum AuthMethod {
+
+ mtls,
+ token,
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java
index f73aeb89f0e..d9384373deb 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentData.java
@@ -8,8 +8,6 @@ import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.athenz.api.AthenzDomain;
import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
-import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
import com.yahoo.vespa.hosted.controller.api.integration.dataplanetoken.DataplaneTokenVersions;
import com.yahoo.vespa.hosted.controller.api.integration.secrets.TenantSecretStore;
import com.yahoo.yolean.concurrent.Memoized;
@@ -18,7 +16,6 @@ import java.io.InputStream;
import java.security.cert.X509Certificate;
import java.util.List;
import java.util.Optional;
-import java.util.Set;
import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
@@ -35,8 +32,7 @@ public class DeploymentData {
private final ZoneId zone;
private final Supplier<InputStream> applicationPackage;
private final Version platform;
- private final Set<ContainerEndpoint> containerEndpoints;
- private final Supplier<Optional<EndpointCertificate>> endpointCertificate;
+ private final Supplier<DeploymentEndpoints> endpoints;
private final Optional<DockerImage> dockerImageRepo;
private final Optional<AthenzDomain> athenzDomain;
private final Supplier<Quota> quota;
@@ -47,8 +43,7 @@ public class DeploymentData {
private final boolean dryRun;
public DeploymentData(ApplicationId instance, ZoneId zone, Supplier<InputStream> applicationPackage, Version platform,
- Set<ContainerEndpoint> containerEndpoints,
- Supplier<Optional<EndpointCertificate>> endpointCertificate,
+ Supplier<DeploymentEndpoints> endpoints,
Optional<DockerImage> dockerImageRepo,
Optional<AthenzDomain> athenzDomain,
Supplier<Quota> quota,
@@ -61,8 +56,7 @@ public class DeploymentData {
this.zone = requireNonNull(zone);
this.applicationPackage = requireNonNull(applicationPackage);
this.platform = requireNonNull(platform);
- this.containerEndpoints = Set.copyOf(requireNonNull(containerEndpoints));
- this.endpointCertificate = new Memoized<>(requireNonNull(endpointCertificate));
+ this.endpoints = new Memoized<>(requireNonNull(endpoints));
this.dockerImageRepo = requireNonNull(dockerImageRepo);
this.athenzDomain = athenzDomain;
this.quota = new Memoized<>(requireNonNull(quota));
@@ -89,12 +83,8 @@ public class DeploymentData {
return platform;
}
- public Set<ContainerEndpoint> containerEndpoints() {
- return containerEndpoints;
- }
-
- public Optional<EndpointCertificate> endpointCertificate() {
- return endpointCertificate.get();
+ public Supplier<DeploymentEndpoints> endpoints() {
+ return endpoints;
}
public Optional<DockerImage> dockerImageRepo() {
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentEndpoints.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentEndpoints.java
new file mode 100644
index 00000000000..9ec17571a35
--- /dev/null
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/application/v4/model/DeploymentEndpoints.java
@@ -0,0 +1,25 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.api.application.v4.model;
+
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
+
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * The endpoints and their certificate (if any) of a deployment.
+ *
+ * @author mpolden
+ */
+public record DeploymentEndpoints(Set<ContainerEndpoint> endpoints, Optional<EndpointCertificate> certificate) {
+
+ public static final DeploymentEndpoints none = new DeploymentEndpoints(Set.of(), Optional.empty());
+
+ public DeploymentEndpoints {
+ Objects.requireNonNull(endpoints);
+ Objects.requireNonNull(certificate);
+ }
+
+}
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java
index 7246903a51b..4746fa2da26 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/configserver/ContainerEndpoint.java
@@ -1,79 +1,39 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.vespa.hosted.controller.api.integration.configserver;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import java.util.List;
+import java.util.Map;
import java.util.Objects;
import java.util.OptionalInt;
/**
- * This represents a list of one or more names for a container cluster.
+ * The endpoint of a container cluster. This encapsulates the endpoint details passed from controller to the config
+ * server on deploy.
+ *
+ * @param clusterId ID of the cluster to which this points
+ * @param scope Scope of this endpoint
+ * @param names All valid DNS names for this endpoint. This can contain both proper DNS names and synthetic identifiers
+ * used for routing, such as a Host header value that is not necessarily a proper DNS name
+ * @param weight The relative weight of this endpoint
+ * @param routingMethod The routing method used by this endpoint
+ * @param authMethods Supported authentication methods for each endpoint name
*
* @author mpolden
*/
-public class ContainerEndpoint {
-
- private final String clusterId;
- private final String scope;
- private final List<String> names;
- private final OptionalInt weight;
- private final RoutingMethod routingMethod;
+public record ContainerEndpoint(String clusterId, String scope, List<String> names, OptionalInt weight,
+ RoutingMethod routingMethod, Map<String, AuthMethod> authMethods) {
- public ContainerEndpoint(String clusterId, String scope, List<String> names, OptionalInt weight, RoutingMethod routingMethod) {
+ public ContainerEndpoint(String clusterId, String scope, List<String> names, OptionalInt weight,
+ RoutingMethod routingMethod, Map<String, AuthMethod> authMethods) {
this.clusterId = nonEmpty(clusterId, "clusterId must be non-empty");
this.scope = Objects.requireNonNull(scope, "scope must be non-null");
this.names = List.copyOf(Objects.requireNonNull(names, "names must be non-null"));
this.weight = Objects.requireNonNull(weight, "weight must be non-null");
this.routingMethod = Objects.requireNonNull(routingMethod, "routingMethod must be non-null");
- }
-
- /** ID of the cluster to which this points */
- public String clusterId() {
- return clusterId;
- }
-
- /** The scope of this endpoint */
- public String scope() {
- return scope;
- }
-
- /**
- * All valid DNS names for this endpoint. This can contain both proper DNS names and synthetic identifiers used for
- * routing, such as a Host header value that is not necessarily a proper DNS name.
- */
- public List<String> names() {
- return names;
- }
-
- /** The relative weight of this endpoint */
- public OptionalInt weight() {
- return weight;
- }
-
- /** The routing method used by this endpoint */
- public RoutingMethod routingMethod() {
- return routingMethod;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- ContainerEndpoint that = (ContainerEndpoint) o;
- return clusterId.equals(that.clusterId) && scope.equals(that.scope) && names.equals(that.names) && weight.equals(that.weight) && routingMethod == that.routingMethod;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(clusterId, scope, names, weight, routingMethod);
- }
-
- @Override
- public String toString() {
- return "container endpoint for cluster " + clusterId + ": " + String.join(", ", names) +
- " [method=" + routingMethod + ",scope=" + scope + ",weight=" +
- weight.stream().boxed().map(Object::toString).findFirst().orElse("<none>") + "]";
+ this.authMethods = Objects.requireNonNull(Map.copyOf(authMethods), "authMethods must be non-null");
}
private static String nonEmpty(String s, String message) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index bac2c0ab9d7..7d785cea736 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -13,6 +13,7 @@ import com.yahoo.config.provision.CloudAccount;
import com.yahoo.config.provision.CloudName;
import com.yahoo.config.provision.DockerImage;
import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.Tags;
import com.yahoo.config.provision.TenantName;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.text.Text;
@@ -28,6 +29,7 @@ import com.yahoo.vespa.flags.ListFlag;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.flags.StringFlag;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentEndpoints;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.identifiers.InstanceId;
import com.yahoo.vespa.hosted.controller.api.integration.billing.BillingController;
@@ -36,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ApplicationReindexing;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.DeploymentResult.LogEntry;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
@@ -55,13 +56,13 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning;
import com.yahoo.vespa.hosted.controller.application.DeploymentQuotaCalculator;
-import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageStream;
import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackageValidator;
+import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml;
import com.yahoo.vespa.hosted.controller.athenz.impl.AthenzFacade;
import com.yahoo.vespa.hosted.controller.certificate.EndpointCertificates;
import com.yahoo.vespa.hosted.controller.concurrent.Once;
@@ -72,6 +73,8 @@ import com.yahoo.vespa.hosted.controller.deployment.Run;
import com.yahoo.vespa.hosted.controller.notification.Notification;
import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpoints;
+import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints;
import com.yahoo.vespa.hosted.controller.security.AccessControl;
import com.yahoo.vespa.hosted.controller.security.Credentials;
import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant;
@@ -89,7 +92,6 @@ import java.security.cert.X509Certificate;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
-import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -515,27 +517,18 @@ public class ApplicationController {
RevisionId revision = run.versions().sourceRevision().filter(__ -> deploySourceVersions).orElse(run.versions().targetRevision());
ApplicationPackageStream applicationPackage = new ApplicationPackageStream(() -> applicationStore.stream(deployment, revision));
AtomicReference<RevisionId> lastRevision = new AtomicReference<>();
- Instance instance;
- Set<ContainerEndpoint> containerEndpoints;
- try (Mutex lock = lock(applicationId)) {
- LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
- application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set);
- instance = application.get().require(job.application().instance());
-
- containerEndpoints = controller.routing().of(deployment).prepare(application);
- } // Release application lock while doing the deployment, which is a lengthy task.
-
- Supplier<Optional<EndpointCertificate>> endpointCertificate = () -> {
+ // Prepare endpoints lazily
+ Supplier<PreparedEndpoints> preparedEndpoints = () -> {
try (Mutex lock = lock(applicationId)) {
- Optional<EndpointCertificate> data = endpointCertificates.get(instance, zone, applicationPackage.truncatedPackage().deploymentSpec());
- data.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version())));
- return data;
+ LockedApplication application = new LockedApplication(requireApplication(applicationId), lock);
+ application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set);
+ return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger);
}
};
// Carry out deployment without holding the application lock.
- DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, containerEndpoints,
- endpointCertificate, run.isDryRun(), run.testerCertificate());
+ DeploymentDataAndResult dataAndResult = deploy(job.application(), applicationPackage, zone, platform, preparedEndpoints,
+ run.isDryRun(), run.testerCertificate());
// Record the quota usage for this application
@@ -572,6 +565,28 @@ public class ApplicationController {
}
}
+ private PreparedEndpoints prepareEndpoints(DeploymentId deployment, JobId job, LockedApplication application,
+ ApplicationPackageStream applicationPackage, Consumer<String> deployLogger) {
+ Instance instance = application.get().require(job.application().instance());
+ Tags tags = applicationPackage.truncatedPackage().deploymentSpec().instance(instance.name())
+ .map(DeploymentInstanceSpec::tags)
+ .orElseGet(Tags::empty);
+ Optional<EndpointCertificate> certificate = endpointCertificates.get(instance, deployment.zoneId(), applicationPackage.truncatedPackage().deploymentSpec());
+ certificate.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version())));
+ BasicServicesXml services;
+ try {
+ services = applicationPackage.truncatedPackage().services(deployment, tags);
+ } catch (Exception e) {
+ // If the basic parsing done by the controller fails, we ignore the exception here so that
+ // complete parsing errors are propagated from the config server. Otherwise, throwing here
+ // will interrupt the request while it's being streamed to the config server
+ log.warning("Ignoring failure to parse services.xml for deployment " + deployment +
+ " while streaming application package: " + Exceptions.toMessageString(e));
+ services = BasicServicesXml.empty;
+ }
+ return controller.routing().of(deployment).prepare(services, certificate, application);
+ }
+
/** Stores the deployment spec and validation overrides from the application package, and runs cleanup. Returns new instances. */
public List<InstanceName> storeWithUpdatedConfig(LockedApplication application, ApplicationPackage applicationPackage) {
validatePackage(applicationPackage, application.get());
@@ -635,7 +650,7 @@ public class ApplicationController {
ApplicationPackageStream applicationPackage = new ApplicationPackageStream(
() -> new ByteArrayInputStream(artifactRepository.getSystemApplicationPackage(application.id(), zone, version))
);
- return deploy(application.id(), applicationPackage, zone, version, Set.of(), Optional::empty, false, Optional.empty()).result();
+ return deploy(application.id(), applicationPackage, zone, version, null, false, Optional.empty()).result();
} else {
throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString());
}
@@ -643,18 +658,18 @@ public class ApplicationController {
/** Deploys the given tester application to the given zone. */
public DeploymentResult deployTester(TesterId tester, ApplicationPackageStream applicationPackage, ZoneId zone, Version platform) {
- return deploy(tester.id(), applicationPackage, zone, platform, Set.of(), Optional::empty, false, Optional.empty()).result();
+ return deploy(tester.id(), applicationPackage, zone, platform, null, false, Optional.empty()).result();
}
private record DeploymentDataAndResult(DeploymentData data, DeploymentResult result) {}
+
private DeploymentDataAndResult deploy(ApplicationId application, ApplicationPackageStream applicationPackage,
- ZoneId zone, Version platform, Set<ContainerEndpoint> endpoints,
- Supplier<Optional<EndpointCertificate>> endpointCertificate,
+ ZoneId zone, Version platform, Supplier<PreparedEndpoints> preparedEndpoints,
boolean dryRun, Optional<X509Certificate> testerCertificate) {
DeploymentId deployment = new DeploymentId(application, zone);
// Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails.
interface CleanCloseable extends AutoCloseable { void close(); }
- List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>();
+ AtomicReference<GeneratedEndpoints> generatedEndpoints = new AtomicReference<>(GeneratedEndpoints.empty);
try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage, generatedEndpoints)) {
Optional<DockerImage> dockerImageRepo = Optional.ofNullable(
dockerImageRepoFlag
@@ -684,26 +699,23 @@ public class ApplicationController {
}
Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec());
List<DataplaneTokenVersions> dataplaneTokenVersions = controller.dataplaneTokenService().listTokens(application.tenant());
- Supplier<Optional<EndpointCertificate>> endpointCertificateWrapper = () -> {
- Optional<EndpointCertificate> data = endpointCertificate.get();
- // TODO(mpolden): Pass these endpoints to config server as part of the deploy call. This will let the
- // application know which endpoints are mTLS and which are token-based
- data.flatMap(EndpointCertificate::randomizedId)
- .ifPresent(applicationPart -> generatedEndpoints.addAll(controller.routing().generateEndpoints(applicationPart, deployment.applicationId())));
- return data;
+ Supplier<DeploymentEndpoints> endpoints = () -> {
+ if (preparedEndpoints == null) return DeploymentEndpoints.none;
+ PreparedEndpoints prepared = preparedEndpoints.get();
+ generatedEndpoints.set(prepared.generatedEndpoints());
+ return new DeploymentEndpoints(prepared.containerEndpoints(), prepared.certificate());
};
DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform,
- endpoints, endpointCertificateWrapper, dockerImageRepo, domain,
- deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun);
+ endpoints, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun);
ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData);
return new DeploymentDataAndResult(deploymentData, preparedApplication.deploymentResult());
}
}
- private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, List<GeneratedEndpoint> generatedEndpoints) {
+ private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, AtomicReference<GeneratedEndpoints> generatedEndpoints) {
if (id.applicationId().instance().isTester()) return;
- controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec(), generatedEndpoints);
+ controller.routing().of(id).activate(data.truncatedPackage().deploymentSpec(), generatedEndpoints.get());
if ( ! id.zoneId().environment().isManuallyDeployed()) return;
controller.applications().applicationStore().putMeta(id, clock.instant(), data.truncatedPackage().metaDataZip());
}
@@ -764,7 +776,7 @@ public class ApplicationController {
}
/**
- * Deletes the the given application. All known instances of the applications will be deleted.
+ * Deletes the given application. All known instances of the applications will be deleted.
*
* @throws IllegalArgumentException if the application has deployments or the caller is not authorized
*/
@@ -784,7 +796,7 @@ public class ApplicationController {
throw new IllegalArgumentException("Could not delete '" + application + "': It has active deployments: " + deployments);
for (Instance instance : application.get().instances().values()) {
- controller.routing().removeEndpointsInDns(application.get(), instance.name());
+ controller.routing().removeRotationEndpointsFromDns(application.get(), instance.name());
application = application.without(instance.name());
}
@@ -820,7 +832,7 @@ public class ApplicationController {
&& application.get().deploymentSpec().instanceNames().contains(instanceId.instance()))
throw new IllegalArgumentException("Can not delete '" + instanceId + "', which is specified in 'deployment.xml'; remove it there first");
- controller.routing().removeEndpointsInDns(application.get(), instanceId.instance());
+ controller.routing().removeRotationEndpointsFromDns(application.get(), instanceId.instance());
curator.writeApplication(application.without(instanceId.instance()).get());
controller.jobController().collectGarbage();
controller.notificationsDb().removeNotifications(NotificationSource.from(instanceId));
@@ -873,7 +885,7 @@ public class ApplicationController {
/**
* Asks the config server whether this deployment is currently healthy, i.e., serving traffic as usual.
- * If this cannot be ascertained, we must assumed it is not.
+ * If this cannot be ascertained, we must assume it is not.
*/
public boolean isHealthy(DeploymentId deploymentId) {
try {
@@ -918,7 +930,7 @@ public class ApplicationController {
DeploymentId id = new DeploymentId(instanceId, zone);
interface CleanCloseable extends AutoCloseable { void close(); }
try (CleanCloseable postDeactivation = () -> {
- application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec(), List.of()));
+ application.ifPresent(app -> controller.routing().of(id).activate(app.get().deploymentSpec(), GeneratedEndpoints.empty));
if (id.zoneId().environment().isManuallyDeployed())
applicationStore.putMetaTombstone(id, clock.instant());
if ( ! id.zoneId().environment().isTest())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index bceef3fd96f..cc6195c075d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -10,6 +10,7 @@ import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.flags.BooleanFlag;
@@ -17,7 +18,6 @@ import com.yahoo.vespa.flags.FetchVector;
import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
@@ -29,8 +29,10 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList;
import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
-import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate;
+import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml;
import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
+import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpoints;
+import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints;
import com.yahoo.vespa.hosted.controller.routing.RoutingId;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicies;
import com.yahoo.vespa.hosted.controller.routing.context.DeploymentRoutingContext;
@@ -51,13 +53,11 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
-import java.util.OptionalInt;
import java.util.Set;
import java.util.TreeMap;
import java.util.stream.Collectors;
@@ -117,22 +117,95 @@ public class RoutingController {
return rotationRepository;
}
- /** Read and return zone-scoped endpoints for given deployment */
+ /** Prepares and returns the endpoints relevant for given deployment */
+ public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) {
+ EndpointList endpoints = EndpointList.EMPTY;
+
+ // Assign rotations to application
+ for (var deploymentInstanceSpec : application.get().deploymentSpec().instances()) {
+ if (deploymentInstanceSpec.concerns(Environment.prod)) {
+ application = controller.routing().assignRotations(application, deploymentInstanceSpec.name());
+ }
+ }
+
+ // Add zone-scoped endpoints
+ final GeneratedEndpoints generatedEndpoints;
+ if (!usesSharedRouting(deployment.zoneId())) { // TODO(mpolden): Remove this check when config models < 8.230 are gone
+ boolean includeTokenEndpoint = tokenEndpointEnabled(deployment.applicationId());
+ Map<ClusterSpec.Id, List<GeneratedEndpoint>> generatedEndpointsByCluster = new HashMap<>();
+ for (var container : services.containers()) {
+ ClusterSpec.Id clusterId = ClusterSpec.Id.from(container.id());
+ boolean tokenSupported = includeTokenEndpoint && container.authMethods().contains(BasicServicesXml.Container.AuthMethod.token);
+ List<GeneratedEndpoint> generatedForCluster = certificate.flatMap(EndpointCertificate::randomizedId)
+ .map(id -> generateEndpoints(id, deployment.applicationId(), tokenSupported))
+ .orElseGet(List::of);
+ if (!generatedForCluster.isEmpty()) {
+ generatedEndpointsByCluster.put(clusterId, generatedForCluster);
+ }
+ endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedForCluster).scope(Scope.zone));
+ }
+ generatedEndpoints = new GeneratedEndpoints(generatedEndpointsByCluster);
+
+ } else {
+ generatedEndpoints = GeneratedEndpoints.empty;
+ }
+
+ // Add global- and application-scoped endpoints
+ endpoints = endpoints.and(declaredEndpointsOf(application.get().id(), application.get().deploymentSpec(), generatedEndpoints).targets(deployment));
+ PreparedEndpoints prepared = new PreparedEndpoints(deployment,
+ endpoints,
+ application.get().require(deployment.applicationId().instance()).rotations(),
+ certificate);
+
+ // Register rotation-backed endpoints in DNS
+ registerRotationEndpointsInDns(prepared);
+
+ return prepared;
+ }
+
+ /** Read and return zone- and region-scoped endpoints for given deployment */
public EndpointList readEndpointsOf(DeploymentId deployment) {
- boolean addTokenEndpoint = tokenEndpointEnabled(deployment.applicationId());
Set<Endpoint> endpoints = new LinkedHashSet<>();
- // To discover the cluster name for a zone-scoped endpoint, we need to read the routing policy
for (var policy : routingPolicies.read(deployment)) {
- RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(policy.id().zone());
- endpoints.addAll(policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint));
- endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod, Optional.empty()));
- for (var ge : policy.generatedEndpoints()) {
- boolean include = switch (ge.authMethod()) {
- case token -> addTokenEndpoint;
- case mtls -> true;
- };
- if (include) {
- endpoints.add(policy.regionEndpointIn(controller.system(), routingMethod, Optional.of(ge)));
+ endpoints.addAll(endpointsOf(deployment, policy.id().cluster(), policy.generatedEndpoints()).asList());
+ }
+ return EndpointList.copyOf(endpoints);
+ }
+
+ /** Returns the zone- and region-scoped endpoints of given deployment */
+ public EndpointList endpointsOf(DeploymentId deployment, ClusterSpec.Id cluster, List<GeneratedEndpoint> generatedEndpoints) {
+ // TODO(mpolden): Support tokens only when generated endpoints are available
+ boolean tokenSupported = tokenEndpointEnabled(deployment.applicationId()) &&
+ (generatedEndpoints.isEmpty() || generatedEndpoints.stream().anyMatch(ge -> ge.authMethod() == AuthMethod.token));
+ RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deployment.zoneId());
+ boolean isProduction = deployment.zoneId().environment().isProduction();
+ List<Endpoint> endpoints = new ArrayList<>();
+ Endpoint.EndpointBuilder zoneEndpoint = Endpoint.of(deployment.applicationId())
+ .routingMethod(routingMethod)
+ .on(Port.fromRoutingMethod(routingMethod))
+ .target(cluster, deployment);
+ endpoints.add(zoneEndpoint.in(controller.system()));
+ if (tokenSupported) {
+ endpoints.add(zoneEndpoint.authMethod(AuthMethod.token).in(controller.system()));
+ }
+ Endpoint.EndpointBuilder regionEndpoint = Endpoint.of(deployment.applicationId())
+ .routingMethod(routingMethod)
+ .on(Port.fromRoutingMethod(routingMethod))
+ .targetRegion(cluster, deployment.zoneId());
+ // Region endpoints are only used by global- and application-endpoints and are thus only needed in
+ // production environments
+ if (isProduction) {
+ endpoints.add(regionEndpoint.in(controller.system()));
+ }
+ for (var generatedEndpoint : generatedEndpoints) {
+ boolean include = switch (generatedEndpoint.authMethod()) {
+ case token -> tokenSupported;
+ case mtls -> true;
+ };
+ if (include) {
+ endpoints.add(zoneEndpoint.generatedFrom(generatedEndpoint).in(controller.system()));
+ if (isProduction) {
+ endpoints.add(regionEndpoint.generatedFrom(generatedEndpoint).in(controller.system()));
}
}
}
@@ -147,43 +220,47 @@ public class RoutingController {
/** Read application and return declared endpoints for given application */
public EndpointList readDeclaredEndpointsOf(TenantAndApplicationId application) {
- return declaredEndpointsOf(controller.applications().requireApplication(application));
+ return readDeclaredEndpointsOf(controller.applications().requireApplication(application));
+ }
+
+ public EndpointList readDeclaredEndpointsOf(Application application) {
+ return declaredEndpointsOf(application.id(), application.deploymentSpec(), readMultiDeploymentGeneratedEndpoints(application.id()));
}
/** Returns endpoints declared in {@link DeploymentSpec} for given application */
- public EndpointList declaredEndpointsOf(Application application) {
- List<GeneratedEndpoint> generatedEndpoints = readGeneratedEndpoints(application);
+ private EndpointList declaredEndpointsOf(TenantAndApplicationId application, DeploymentSpec deploymentSpec, GeneratedEndpoints generatedEndpoints) {
Set<Endpoint> endpoints = new LinkedHashSet<>();
- DeploymentSpec deploymentSpec = application.deploymentSpec();
+ // Global endpoints
for (var spec : deploymentSpec.instances()) {
- ApplicationId instance = application.id().instance(spec.name());
- // Add endpoints declared with current syntax
+ ApplicationId instance = application.instance(spec.name());
spec.endpoints().forEach(declaredEndpoint -> {
RoutingId routingId = RoutingId.of(instance, EndpointId.of(declaredEndpoint.endpointId()));
List<DeploymentId> deployments = declaredEndpoint.regions().stream()
.map(region -> new DeploymentId(instance,
ZoneId.from(Environment.prod, region)))
.toList();
- endpoints.addAll(computeGlobalEndpoints(routingId, ClusterSpec.Id.from(declaredEndpoint.containerId()), deployments, generatedEndpoints));
+ ClusterSpec.Id cluster = ClusterSpec.Id.from(declaredEndpoint.containerId());
+ endpoints.addAll(computeGlobalEndpoints(routingId, cluster, deployments, generatedEndpoints));
});
}
- // Add application endpoints
+ // Application endpoints
for (var declaredEndpoint : deploymentSpec.endpoints()) {
Map<DeploymentId, Integer> deployments = declaredEndpoint.targets().stream()
- .collect(toMap(t -> new DeploymentId(application.id().instance(t.instance()),
+ .collect(toMap(t -> new DeploymentId(application.instance(t.instance()),
ZoneId.from(Environment.prod, t.region())),
t -> t.weight()));
ZoneId zone = deployments.keySet().iterator().next().zoneId(); // Where multiple zones are possible, they all have the same routing method.
RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive;
- Endpoint.EndpointBuilder builder = Endpoint.of(application.id())
+ ClusterSpec.Id cluster = ClusterSpec.Id.from(declaredEndpoint.containerId());
+ Endpoint.EndpointBuilder builder = Endpoint.of(application)
.targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
- ClusterSpec.Id.from(declaredEndpoint.containerId()),
+ cluster,
deployments)
.routingMethod(routingMethod)
.on(Port.fromRoutingMethod(routingMethod));
endpoints.add(builder.in(controller.system()));
- for (var ge : generatedEndpoints) {
+ for (var ge : generatedEndpoints.cluster(cluster)) {
endpoints.add(builder.generatedFrom(ge).in(controller.system()));
}
}
@@ -195,6 +272,7 @@ public class RoutingController {
TreeMap<ZoneId, List<Endpoint>> endpoints = new TreeMap<>(Comparator.comparing(ZoneId::value));
for (var deployment : deployments) {
EndpointList zoneEndpoints = readEndpointsOf(deployment).scope(Endpoint.Scope.zone)
+ .authMethod(AuthMethod.mtls)
.not().legacy();
EndpointList directEndpoints = zoneEndpoints.direct();
if (!directEndpoints.isEmpty()) {
@@ -255,62 +333,47 @@ public class RoutingController {
return Collections.unmodifiableList(endpointDnsNames);
}
- /** Returns the global and application-level endpoints for given deployment, as container endpoints */
- public Set<ContainerEndpoint> containerEndpointsOf(LockedApplication application, InstanceName instanceName, ZoneId zone) {
- // Assign rotations to application
- for (var deploymentInstanceSpec : application.get().deploymentSpec().instances()) {
- if (deploymentInstanceSpec.concerns(Environment.prod)) {
- application = controller.routing().assignRotations(application, deploymentInstanceSpec.name());
- }
+ /** Remove endpoints in DNS for all rotations assigned to given instance */
+ public void removeRotationEndpointsFromDns(Application application, InstanceName instanceName) {
+ Set<Endpoint> endpointsToRemove = new LinkedHashSet<>();
+ Instance instance = application.require(instanceName);
+ // Compute endpoints from rotations. When removing DNS records for rotation-based endpoints we cannot use the
+ // deployment spec, because submitting an empty deployment spec is the first step of removing an application
+ for (var rotation : instance.rotations()) {
+ var deployments = rotation.regions().stream()
+ .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region)))
+ .toList();
+ endpointsToRemove.addAll(computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()),
+ rotation.clusterId(), deployments, readMultiDeploymentGeneratedEndpoints(application.id())));
}
+ endpointsToRemove.forEach(endpoint -> controller.nameServiceForwarder()
+ .removeRecords(Record.Type.CNAME,
+ RecordName.from(endpoint.dnsName()),
+ Priority.normal,
+ Optional.of(application.id())));
+ }
- // Add endpoints backed by a rotation, and register them in DNS if necessary
- Instance instance = application.get().require(instanceName);
- Set<ContainerEndpoint> containerEndpoints = new HashSet<>();
- DeploymentId deployment = new DeploymentId(instance.id(), zone);
- EndpointList endpoints = declaredEndpointsOf(application.get()).targets(deployment);
- EndpointList globalEndpoints = endpoints.scope(Endpoint.Scope.global);
- for (var assignedRotation : instance.rotations()) {
+ private void registerRotationEndpointsInDns(PreparedEndpoints prepared) {
+ TenantAndApplicationId owner = TenantAndApplicationId.from(prepared.deployment().applicationId());
+ EndpointList globalEndpoints = prepared.endpoints().scope(Scope.global);
+ for (var assignedRotation : prepared.rotations()) {
EndpointList rotationEndpoints = globalEndpoints.named(assignedRotation.endpointId(), Scope.global)
.requiresRotation();
-
// Skip rotations which do not apply to this zone
- if (!assignedRotation.regions().contains(zone.region())) {
+ if (!assignedRotation.regions().contains(prepared.deployment().zoneId().region())) {
continue;
}
-
// Register names in DNS
Rotation rotation = rotationRepository.requireRotation(assignedRotation.rotationId());
for (var endpoint : rotationEndpoints) {
controller.nameServiceForwarder().createRecord(
new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(rotation.name())),
Priority.normal,
- Optional.of(application.get().id()));
- List<String> names = List.of(endpoint.dnsName(),
- // Include rotation ID as a valid name of this container endpoint
- // (required by global routing health checks)
- assignedRotation.rotationId().asString());
- containerEndpoints.add(new ContainerEndpoint(assignedRotation.clusterId().value(),
- asString(Endpoint.Scope.global),
- names,
- OptionalInt.empty(),
- endpoint.routingMethod()));
+ Optional.of(owner)
+ );
}
}
- // Add endpoints not backed by a rotation (i.e. other routing methods so that the config server always knows
- // about global names, even when not using rotations)
- globalEndpoints.not().requiresRotation()
- .groupingBy(Endpoint::cluster)
- .forEach((clusterId, clusterEndpoints) -> {
- containerEndpoints.add(new ContainerEndpoint(clusterId.value(),
- asString(Endpoint.Scope.global),
- clusterEndpoints.mapToList(Endpoint::dnsName),
- OptionalInt.empty(),
- RoutingMethod.exclusive));
- });
- // Add application endpoints
- EndpointList applicationEndpoints = endpoints.scope(Endpoint.Scope.application);
- for (var endpoint : applicationEndpoints.shared()) { // DNS for non-shared endpoints is handled by RoutingPolicies
+ for (var endpoint : prepared.endpoints().scope(Scope.application).shared()) { // DNS for non-shared application endpoints is handled by RoutingPolicies
Set<ZoneId> targetZones = endpoint.targets().stream()
.map(t -> t.deployment().zoneId())
.collect(Collectors.toUnmodifiableSet());
@@ -323,79 +386,31 @@ public class RoutingController {
controller.nameServiceForwarder().createRecord(
new Record(Record.Type.CNAME, RecordName.from(endpoint.dnsName()), RecordData.fqdn(vipHostname)),
Priority.normal,
- Optional.of(application.get().id()));
- }
- Map<ClusterSpec.Id, EndpointList> applicationEndpointsByCluster = applicationEndpoints.groupingBy(Endpoint::cluster);
- for (var kv : applicationEndpointsByCluster.entrySet()) {
- ClusterSpec.Id clusterId = kv.getKey();
- EndpointList clusterEndpoints = kv.getValue();
- for (var endpoint : clusterEndpoints) {
- Optional<Endpoint.Target> matchingTarget = endpoint.targets().stream()
- .filter(t -> t.routesTo(deployment))
- .findFirst();
- if (matchingTarget.isEmpty()) throw new IllegalStateException("No target found routing to " + deployment + " in " + endpoint);
- containerEndpoints.add(new ContainerEndpoint(clusterId.value(),
- asString(Endpoint.Scope.application),
- List.of(endpoint.dnsName()),
- OptionalInt.of(matchingTarget.get().weight()),
- endpoint.routingMethod()));
- }
+ Optional.of(owner));
}
- return Collections.unmodifiableSet(containerEndpoints);
- }
-
- /** Remove endpoints in DNS for all rotations assigned to given instance */
- public void removeEndpointsInDns(Application application, InstanceName instanceName) {
- Set<Endpoint> endpointsToRemove = new LinkedHashSet<>();
- Instance instance = application.require(instanceName);
- // Compute endpoints from rotations. When removing DNS records for rotation-based endpoints we cannot use the
- // deployment spec, because submitting an empty deployment spec is the first step of removing an application
- for (var rotation : instance.rotations()) {
- var deployments = rotation.regions().stream()
- .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region)))
- .toList();
- endpointsToRemove.addAll(computeGlobalEndpoints(RoutingId.of(instance.id(), rotation.endpointId()),
- rotation.clusterId(), deployments, readGeneratedEndpoints(application)));
- }
- endpointsToRemove.forEach(endpoint -> controller.nameServiceForwarder()
- .removeRecords(Record.Type.CNAME,
- RecordName.from(endpoint.dnsName()),
- Priority.normal,
- Optional.of(application.id())));
}
/** Generate endpoints for all authentication methods, using given application part */
- public List<GeneratedEndpoint> generateEndpoints(String applicationPart, ApplicationId instance) {
+ private List<GeneratedEndpoint> generateEndpoints(String applicationPart, ApplicationId instance, boolean token) {
if (!randomizedEndpointsEnabled(instance)) {
return List.of();
}
- return generateEndpoints(applicationPart);
- }
-
-
- private List<GeneratedEndpoint> generateEndpoints(String applicationPart) {
- return Arrays.stream(Endpoint.AuthMethod.values())
+ return Arrays.stream(AuthMethod.values())
+ .filter(method -> method != AuthMethod.token || token)
.map(method -> new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)),
applicationPart,
method))
.toList();
}
- /** This is only suitable for use in declared endpoints, which ignore the randomly generated cluster part */
- private List<GeneratedEndpoint> readGeneratedEndpoints(Application application) {
- boolean includeTokenEndpoint = application.productionInstances().values().stream()
- .map(Instance::id)
- .anyMatch(this::tokenEndpointEnabled);
- Optional<String> randomizedId = controller.curator().readAssignedCertificate(application.id(), Optional.empty())
- .map(AssignedCertificate::certificate)
- .flatMap(EndpointCertificate::randomizedId);
- if (randomizedId.isEmpty()) {
- return List.of();
+ /** Returns generated endpoint suitable for use in endpoints whose scope is {@link Scope#multiDeployment()} */
+ private GeneratedEndpoints readMultiDeploymentGeneratedEndpoints(TenantAndApplicationId application) {
+ Map<ClusterSpec.Id, List<GeneratedEndpoint>> endpoints = new HashMap<>();
+ for (var policy : policies().read(application)) {
+ // The cluster part is not used in this context because multi-deployment endpoints have a user-controlled name
+ endpoints.putIfAbsent(policy.id().cluster(), policy.generatedEndpoints().stream().toList());
}
- return generateEndpoints(randomizedId.get()).stream().filter(endpoint -> switch (endpoint.authMethod()) {
- case token -> includeTokenEndpoint;
- case mtls -> true;
- }).toList();
+ return new GeneratedEndpoints(endpoints);
}
/**
@@ -435,7 +450,7 @@ public class RoutingController {
}
/** Compute global endpoints for given routing ID, application and deployments */
- private List<Endpoint> computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, List<GeneratedEndpoint> generatedEndpoints) {
+ private List<Endpoint> computeGlobalEndpoints(RoutingId routingId, ClusterSpec.Id cluster, List<DeploymentId> deployments, GeneratedEndpoints generatedEndpoints) {
var endpoints = new ArrayList<Endpoint>();
var directMethods = 0;
var availableRoutingMethods = routingMethodsOfAll(deployments);
@@ -449,14 +464,15 @@ public class RoutingController {
.on(Port.fromRoutingMethod(method))
.routingMethod(method);
endpoints.add(builder.in(controller.system()));
- for (var ge : generatedEndpoints) {
+ for (var ge : generatedEndpoints.cluster(cluster)) {
endpoints.add(builder.generatedFrom(ge).in(controller.system()));
}
}
return endpoints;
}
- public boolean tokenEndpointEnabled(ApplicationId instance) {
+
+ private boolean tokenEndpointEnabled(ApplicationId instance) {
return createTokenEndpoint.with(FetchVector.Dimension.APPLICATION_ID, instance.serializedForm()).value();
}
@@ -472,13 +488,5 @@ public class RoutingController {
return 'v' + base32 + Endpoint.internalDnsSuffix(system);
}
- private static String asString(Endpoint.Scope scope) {
- return switch (scope) {
- case application -> "application";
- case global -> "global";
- case weighted -> "weighted";
- case zone -> "zone";
- };
- }
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 1a4095001ff..010bc023dad 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.text.Text;
@@ -165,7 +166,7 @@ public class Endpoint {
@Override
public String toString() {
- return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s]", url, scope, legacy, routingMethod, authMethod);
+ return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s, name=%s]", url, scope, legacy, routingMethod, authMethod, name());
}
private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) {
@@ -405,12 +406,6 @@ public class Endpoint {
}
- /** An endpoint's authentication method */
- public enum AuthMethod {
- mtls,
- token,
- }
-
/** Represents an endpoint's HTTP port */
public record Port(int port) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
index dcc3e229f92..310a78e45f0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application;
import com.yahoo.collections.AbstractFilteringList;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import java.util.Collection;
@@ -94,10 +95,19 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
return matching(endpoint -> endpoint.routingMethod().isShared());
}
+ /** Returns the subset of endpoints supporting given authentication method */
+ public EndpointList authMethod(AuthMethod authMethod) {
+ return matching(endpoint -> endpoint.authMethod() == authMethod);
+ }
+
public static EndpointList copyOf(Collection<Endpoint> endpoints) {
return new EndpointList(endpoints, false);
}
+ public static EndpointList of(Endpoint ...endpoint) {
+ return copyOf(List.of(endpoint));
+ }
+
@Override
public String toString() {
return asList().toString();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java
index dd6f4e5111d..a9d6dcb08f9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java
@@ -1,6 +1,7 @@
package com.yahoo.vespa.hosted.controller.application;
import ai.vespa.validation.Validation;
+import com.yahoo.config.provision.zone.AuthMethod;
import java.util.random.RandomGenerator;
import java.util.regex.Pattern;
@@ -11,7 +12,7 @@ import java.util.regex.Pattern;
*
* @author mpolden
*/
-public record GeneratedEndpoint(String clusterPart, String applicationPart, Endpoint.AuthMethod authMethod) {
+public record GeneratedEndpoint(String clusterPart, String applicationPart, AuthMethod authMethod) {
private static final Pattern PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
index 3ec79b03ee8..3ec7f120726 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackage.java
@@ -22,8 +22,11 @@ import com.yahoo.vespa.archive.ArchiveStreamReader;
import com.yahoo.vespa.archive.ArchiveStreamReader.ArchiveFile;
import com.yahoo.vespa.archive.ArchiveStreamReader.Options;
import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.deployment.ZipBuilder;
import com.yahoo.yolean.Exceptions;
+import org.w3c.dom.Document;
+
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
@@ -136,6 +139,25 @@ public class ApplicationPackage {
*/
public ValidationOverrides validationOverrides() { return validationOverrides; }
+ /** Returns a basic variant of services.xml contained in this package, pre-processed according to given deployment and tags */
+ public BasicServicesXml services(DeploymentId deployment, Tags tags) {
+ FileWrapper servicesXml = files.wrapper().wrap(Paths.get(servicesFile));
+ if (!servicesXml.exists()) return BasicServicesXml.empty;
+ try {
+ Document document = new XmlPreProcessor(files.wrapper().wrap(Paths.get("./")),
+ new InputStreamReader(new ByteArrayInputStream(servicesXml.content()), UTF_8),
+ deployment.applicationId().instance(),
+ deployment.zoneId().environment(),
+ deployment.zoneId().region(),
+ tags).run();
+ return BasicServicesXml.parse(document);
+ } catch (IllegalArgumentException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+
/** Returns the platform version which package was compiled against, if known. */
public Optional<Version> compileVersion() { return compileVersion; }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java
new file mode 100644
index 00000000000..9eb10857526
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXml.java
@@ -0,0 +1,92 @@
+package com.yahoo.vespa.hosted.controller.application.pkg;
+
+import com.yahoo.text.XML;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+/**
+ * A partially parsed variant of services.xml, for use by the {@link com.yahoo.vespa.hosted.controller.Controller}.
+ *
+ * @author mpolden
+ */
+public record BasicServicesXml(List<Container> containers) {
+
+ public static final BasicServicesXml empty = new BasicServicesXml(List.of());
+
+ private static final String SERVICES_TAG = "services";
+ private static final String CONTAINER_TAG = "container";
+ private static final String CLIENTS_TAG = "clients";
+ private static final String CLIENT_TAG = "client";
+ private static final String TOKEN_TAG = "token";
+
+ public BasicServicesXml(List<Container> containers) {
+ this.containers = List.copyOf(Objects.requireNonNull(containers));
+ }
+
+ /** Parse a services.xml from given document */
+ public static BasicServicesXml parse(Document document) {
+ Element root = document.getDocumentElement();
+ if (!root.getTagName().equals("services")) {
+ throw new IllegalArgumentException("Root tag must be <" + SERVICES_TAG + ">");
+ }
+ List<BasicServicesXml.Container> containers = new ArrayList<>();
+ for (var childNode : XML.getChildren(root)) {
+ if (childNode.getTagName().equals(CONTAINER_TAG)) {
+ String id = childNode.getAttribute("id");
+ if (id.isEmpty()) throw new IllegalArgumentException(CONTAINER_TAG + " tag requires 'id' attribute");
+ List<Container.AuthMethod> methods = parseAuthMethods(childNode);
+ containers.add(new Container(id, methods));
+ }
+ }
+ return new BasicServicesXml(containers);
+ }
+
+ private static List<BasicServicesXml.Container.AuthMethod> parseAuthMethods(Element containerNode) {
+ List<BasicServicesXml.Container.AuthMethod> methods = new ArrayList<>();
+ for (var node : XML.getChildren(containerNode)) {
+ if (node.getTagName().equals(CLIENTS_TAG)) {
+ for (var clientNode : XML.getChildren(node)) {
+ if (clientNode.getTagName().equals(CLIENT_TAG)) {
+ boolean tokenEnabled = XML.getChildren(clientNode).stream()
+ .anyMatch(n -> n.getTagName().equals(TOKEN_TAG));
+ methods.add(tokenEnabled ? Container.AuthMethod.token : Container.AuthMethod.mtls);
+ }
+ }
+ }
+ }
+ if (methods.isEmpty()) {
+ methods.add(Container.AuthMethod.mtls);
+ }
+ return methods;
+ }
+
+ /**
+ * A Vespa container service.
+ *
+ * @param id ID of container
+ * @param authMethods Authentication methods supported by this container
+ */
+ public record Container(String id, List<AuthMethod> authMethods) {
+
+ public Container(String id, List<AuthMethod> authMethods) {
+ this.id = Objects.requireNonNull(id);
+ this.authMethods = Objects.requireNonNull(authMethods).stream()
+ .distinct()
+ .sorted()
+ .collect(Collectors.toList());
+ if (authMethods.isEmpty()) throw new IllegalArgumentException("Container must have at least one auth method");
+ }
+
+ public enum AuthMethod {
+ mtls,
+ token,
+ }
+
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
index 5770649c8b7..9bc9403b9d6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
@@ -4,13 +4,13 @@ package com.yahoo.vespa.hosted.controller.persistence;
import ai.vespa.http.DomainName;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.slime.ArrayTraverser;
import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
import com.yahoo.slime.SlimeUtils;
-import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
@@ -132,17 +132,17 @@ public class RoutingPolicySerializer {
return new RoutingStatus(status, agent, changedAt);
}
- private String authMethod(Endpoint.AuthMethod authMethod) {
+ private String authMethod(AuthMethod authMethod) {
return switch (authMethod) {
case token -> "token";
case mtls -> "mtls";
};
}
- private Endpoint.AuthMethod authMethodFromSlime(Inspector field) {
+ private AuthMethod authMethodFromSlime(Inspector field) {
return switch (field.asString()) {
- case "token" -> Endpoint.AuthMethod.token;
- case "mtls" -> Endpoint.AuthMethod.mtls;
+ case "token" -> AuthMethod.token;
+ case "mtls" -> AuthMethod.mtls;
default -> throw new IllegalArgumentException("Unknown auth method '" + field.asString() + "'");
};
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 9394a1fcbe2..a0e8b1c5610 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -333,7 +333,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
List<EndpointTarget> targets = new ArrayList<>();
out:
for (var app : applications) {
- Optional<Endpoint> declaredEndpoint = controller.routing().declaredEndpointsOf(app).dnsName(endpoint);
+ Optional<Endpoint> declaredEndpoint = controller.routing().readDeclaredEndpointsOf(app).dnsName(endpoint);
if (declaredEndpoint.isPresent()) {
for (var target : declaredEndpoint.get().targets()) {
targets.add(new EndpointTarget(target.deployment(), declaredEndpoint.get().cluster()));
@@ -2061,7 +2061,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
if (!legacyEndpoints) {
zoneEndpoints = zoneEndpoints.not().legacy().direct();
}
- EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application).targets(deploymentId);
+ EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application).targets(deploymentId);
if (!legacyEndpoints) {
declaredEndpoints = declaredEndpoints.not().legacy().direct();
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index 232f25f5674..bc83eeb73c1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -253,7 +253,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
var instances = instanceId == null
? application.instances().values()
: List.of(application.require(instanceId.instance()));
- EndpointList declaredEndpoints = controller.routing().declaredEndpointsOf(application);
+ EndpointList declaredEndpoints = controller.routing().readDeclaredEndpointsOf(application);
for (var instance : instances) {
var zones = zoneId == null
? instance.deployments().keySet().stream().sorted(Comparator.comparing(ZoneId::value)).toList()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java
new file mode 100644
index 00000000000..3adbb43a7b5
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/GeneratedEndpoints.java
@@ -0,0 +1,32 @@
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+/**
+ * This represents endpoints generated by the controller for a deployment, grouped by their assigned cluster.
+ *
+ * @author mpolden
+ */
+public record GeneratedEndpoints(Map<ClusterSpec.Id, List<GeneratedEndpoint>> endpoints) {
+
+ public static final GeneratedEndpoints empty = new GeneratedEndpoints(Map.of());
+
+ public GeneratedEndpoints(Map<ClusterSpec.Id, List<GeneratedEndpoint>> endpoints) {
+ this.endpoints = Map.copyOf(Objects.requireNonNull(endpoints));
+ endpoints.forEach((cluster, generatedEndpoints) -> {
+ if (generatedEndpoints.stream().distinct().count() != generatedEndpoints.size()) {
+ throw new IllegalStateException("Endpoints for " + cluster + " must be distinct, got " + generatedEndpoints);
+ }
+ });
+ }
+
+ public List<GeneratedEndpoint> cluster(ClusterSpec.Id cluster) {
+ return endpoints.getOrDefault(cluster, List.of());
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java
new file mode 100644
index 00000000000..c67d88fa81f
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java
@@ -0,0 +1,121 @@
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.zone.AuthMethod;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
+import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.Set;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * This represents the endpoints, and associated resources, that have been prepared for a deployment.
+ *
+ * @author mpolden
+ */
+public record PreparedEndpoints(DeploymentId deployment,
+ EndpointList endpoints,
+ List<AssignedRotation> rotations,
+ Optional<EndpointCertificate> certificate) {
+
+ public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, Optional<EndpointCertificate> certificate) {
+ this.deployment = Objects.requireNonNull(deployment);
+ this.endpoints = Objects.requireNonNull(endpoints);
+ this.rotations = List.copyOf(Objects.requireNonNull(rotations));
+ this.certificate = Objects.requireNonNull(certificate);
+ }
+
+ /** Returns the endpoints generated by this prepare */
+ public GeneratedEndpoints generatedEndpoints() {
+ Map<ClusterSpec.Id, List<GeneratedEndpoint>> generated = new HashMap<>();
+ for (var endpoint : endpoints.generated()) {
+ List<GeneratedEndpoint> clusterGenerated = generated.computeIfAbsent(endpoint.cluster(), (k) -> new ArrayList<>());
+ if (!clusterGenerated.contains(endpoint.generated().get())) {
+ clusterGenerated.add(endpoint.generated().get());
+ }
+ }
+ return new GeneratedEndpoints(generated);
+ }
+
+ /** Returns the endpoints contained in this as {@link com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint} */
+ public Set<ContainerEndpoint> containerEndpoints() {
+ Map<EndpointId, AssignedRotation> rotationsByEndpointId = rotations.stream()
+ .collect(Collectors.toMap(AssignedRotation::endpointId,
+ Function.identity()));
+ Set<ContainerEndpoint> containerEndpoints = new HashSet<>();
+ endpoints.scope(Endpoint.Scope.zone).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> {
+ containerEndpoints.add(new ContainerEndpoint(clusterId.value(),
+ asString(Endpoint.Scope.zone),
+ clusterEndpoints.mapToList(Endpoint::dnsName),
+ OptionalInt.empty(),
+ clusterEndpoints.first().get().routingMethod(),
+ authMethodsByDnsName(clusterEndpoints)));
+ });
+ endpoints.scope(Endpoint.Scope.global).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> {
+ for (var endpoint : clusterEndpoints) {
+ List<String> names = new ArrayList<>(2);
+ names.add(endpoint.dnsName());
+ if (endpoint.requiresRotation()) {
+ EndpointId endpointId = EndpointId.of(endpoint.name());
+ AssignedRotation rotation = rotationsByEndpointId.get(endpointId);
+ if (rotation == null) {
+ throw new IllegalArgumentException(endpoint + " requires a rotation, but no rotation has been assigned to " + endpointId);
+ }
+ // Include the rotation ID as a valid name of this container endpoint
+ // (required by global routing health checks)
+ names.add(rotation.rotationId().asString());
+ }
+ containerEndpoints.add(new ContainerEndpoint(clusterId.value(),
+ asString(Endpoint.Scope.global),
+ names,
+ OptionalInt.empty(),
+ endpoint.routingMethod(),
+ authMethodsByDnsName(EndpointList.of(endpoint))));
+ }
+ });
+ endpoints.scope(Endpoint.Scope.application).groupingBy(Endpoint::cluster).forEach((clusterId, clusterEndpoints) -> {
+ for (var endpoint : clusterEndpoints) {
+ Optional<Endpoint.Target> matchingTarget = endpoint.targets().stream()
+ .filter(t -> t.routesTo(deployment))
+ .findFirst();
+ if (matchingTarget.isEmpty()) throw new IllegalStateException("No target found routing to " + deployment + " in " + endpoint);
+ containerEndpoints.add(new ContainerEndpoint(clusterId.value(),
+ asString(Endpoint.Scope.application),
+ List.of(endpoint.dnsName()),
+ OptionalInt.of(matchingTarget.get().weight()),
+ endpoint.routingMethod(),
+ authMethodsByDnsName(EndpointList.of(endpoint))));
+ }
+ });
+ return containerEndpoints;
+ }
+
+ private static Map<String, AuthMethod> authMethodsByDnsName(EndpointList endpoints) {
+ return endpoints.asList().stream().collect(Collectors.toMap(Endpoint::dnsName, Endpoint::authMethod));
+ }
+
+ private static String asString(Endpoint.Scope scope) {
+ return switch (scope) {
+ case application -> "application";
+ case global -> "global";
+ case weighted -> "weighted";
+ case zone -> "zone";
+ };
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index c8c3d057ee3..eb881519589 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.routing;
import ai.vespa.http.DomainName;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.transaction.Mutex;
@@ -43,6 +44,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Function;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
@@ -107,7 +109,7 @@ public class RoutingPolicies {
* Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if
* routing configuration affecting given deployment has changed.
*/
- public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) {
+ public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, GeneratedEndpoints generatedEndpoints) {
ApplicationId instance = deployment.applicationId();
List<LoadBalancer> loadBalancers = controller.serviceRegistry().configServer()
.getLoadBalancers(instance, deployment.zoneId());
@@ -243,14 +245,25 @@ public class RoutingPolicies {
for (var policy : policies) {
if (policy.dnsZone().isEmpty() && policy.canonicalName().isPresent()) continue;
if (controller.zoneRegistry().routingMethod(policy.id().zone()) != RoutingMethod.exclusive) continue;
- Endpoint endpoint = policy.regionEndpointIn(controller.system(), RoutingMethod.exclusive, parent.generated());
var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
- long weight = 1;
- if (isConfiguredOut(zonePolicy, policy)) {
- weight = 0; // A record with 0 weight will not receive traffic. If all records within a group have 0
- // weight, traffic is routed to all records with equal probability.
+ // A record with 0 weight will not receive traffic. If all records within a group have 0
+ // weight, traffic is routed to all records with equal probability
+ long weight = isConfiguredOut(zonePolicy, policy) ? 0 : 1;
+ boolean generated = parent.generated().isPresent();
+ EndpointList weightedEndpoints = controller.routing()
+ .endpointsOf(policy.id().deployment(),
+ policy.id().cluster(),
+ parent.generated().stream().toList())
+ .scope(Endpoint.Scope.weighted);
+ if (generated) {
+ weightedEndpoints = weightedEndpoints.generated();
+ } else {
+ weightedEndpoints = weightedEndpoints.not().generated();
}
-
+ if (weightedEndpoints.size() != 1) {
+ throw new IllegalStateException("Expected to compute exactly one region endpoint for " + policy.id() + " with parent " + parent);
+ }
+ Endpoint endpoint = weightedEndpoints.first().get();
RegionEndpoint regionEndpoint = endpoints.computeIfAbsent(endpoint, (k) -> new RegionEndpoint(
new LatencyAliasTarget(DomainName.of(endpoint.dnsName()), policy.dnsZone().get(), policy.id().zone())));
@@ -282,7 +295,7 @@ public class RoutingPolicies {
Map<Endpoint, Set<Target>> inactiveTargetsByEndpoint = new LinkedHashMap<>();
for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
RoutingId routingId = routeEntry.getKey();
- EndpointList endpoints = controller.routing().declaredEndpointsOf(application)
+ EndpointList endpoints = controller.routing().readDeclaredEndpointsOf(application)
.named(routingId.endpointId(), Endpoint.Scope.application);
for (Endpoint endpoint : endpoints) {
for (var policy : routeEntry.getValue()) {
@@ -355,22 +368,23 @@ public class RoutingPolicies {
*
* @return the updated policies
*/
- private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, List<GeneratedEndpoint> generatedEndpoints, @SuppressWarnings("unused") Mutex lock) {
+ private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, GeneratedEndpoints generatedEndpoints, @SuppressWarnings("unused") Mutex lock) {
Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(applicationPolicies.instance(allocation.deployment.applicationId()).asMap());
for (LoadBalancer loadBalancer : allocation.loadBalancers) {
if (loadBalancer.hostname().isEmpty() && loadBalancer.ipAddress().isEmpty()) continue;
var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId());
var existingPolicy = policies.get(policyId);
var dnsZone = loadBalancer.ipAddress().isPresent() ? Optional.of("ignored") : loadBalancer.dnsZone();
+ var clusterGeneratedEndpoints = generatedEndpoints.cluster(loadBalancer.cluster());
var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname(), loadBalancer.ipAddress(), dnsZone,
allocation.instanceEndpointsOf(loadBalancer),
allocation.applicationEndpointsOf(loadBalancer),
RoutingStatus.DEFAULT,
loadBalancer.isPublic(),
- generatedEndpoints);
- boolean addingGeneratedEndpoints = !generatedEndpoints.isEmpty() && (existingPolicy == null || existingPolicy.generatedEndpoints().isEmpty());
+ clusterGeneratedEndpoints);
+ boolean addingGeneratedEndpoints = !clusterGeneratedEndpoints.isEmpty() && (existingPolicy == null || existingPolicy.generatedEndpoints().isEmpty());
if (addingGeneratedEndpoints) {
- generatedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies));
+ clusterGeneratedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies));
}
if (existingPolicy != null) {
newPolicy = newPolicy.with(existingPolicy.routingStatus()); // Always preserve routing status
@@ -386,11 +400,17 @@ public class RoutingPolicies {
return updated;
}
+ private static Map<AuthMethod, GeneratedEndpoint> asMap(List<GeneratedEndpoint> generatedEndpoints) {
+ return generatedEndpoints.stream().collect(Collectors.toMap(GeneratedEndpoint::authMethod, Function.identity()));
+ }
+
/** Update zone DNS record for given policy */
private void updateZoneDnsOf(RoutingPolicy policy, LoadBalancer loadBalancer, DeploymentId deploymentId) {
- RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(deploymentId.zoneId());
- boolean addTokenEndpoint = controller.routing().tokenEndpointEnabled(deploymentId.applicationId());
- for (var endpoint : policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)) {
+ EndpointList zoneEndpoints = controller.routing().endpointsOf(deploymentId,
+ policy.id().cluster(),
+ policy.generatedEndpoints())
+ .scope(Endpoint.Scope.zone);
+ for (var endpoint : zoneEndpoints) {
RecordName name = RecordName.from(endpoint.dnsName());
Record record = policy.canonicalName().isPresent() ?
new Record(Record.Type.CNAME, name, RecordData.fqdn(policy.canonicalName().get().value())) :
@@ -464,14 +484,16 @@ public class RoutingPolicies {
* @return the updated policies
*/
private RoutingPolicyList removePoliciesUnreferencedBy(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) {
- RoutingMethod routingMethod = controller.zoneRegistry().routingMethod(allocation.deployment.zoneId());
- boolean addTokenEndpoint = controller.routing().tokenEndpointEnabled(allocation.deployment.applicationId());
Map<RoutingPolicyId, RoutingPolicy> newPolicies = new LinkedHashMap<>(instancePolicies.asMap());
Set<RoutingPolicyId> activeIds = allocation.asPolicyIds();
RoutingPolicyList removable = instancePolicies.deployment(allocation.deployment)
.not().matching(policy -> activeIds.contains(policy.id()));
for (var policy : removable) {
- for (var endpoint : policy.zoneEndpointsIn(controller.system(), routingMethod, addTokenEndpoint)) {
+ EndpointList zoneEndpoints = controller.routing().endpointsOf(allocation.deployment,
+ policy.id().cluster(),
+ policy.generatedEndpoints())
+ .scope(Endpoint.Scope.zone);
+ for (var endpoint : zoneEndpoints) {
Record.Type type = policy.canonicalName().isPresent() ? Record.Type.CNAME : Record.Type.A;
nameServiceForwarder(endpoint).removeRecords(type,
RecordName.from(endpoint.dnsName()),
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index 0233e7502ef..2363524e306 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -3,15 +3,10 @@ package com.yahoo.vespa.hosted.controller.routing;
import ai.vespa.http.DomainName;
import com.google.common.collect.ImmutableSortedSet;
-import com.yahoo.config.provision.SystemName;
-import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
-import com.yahoo.vespa.hosted.controller.application.Endpoint;
-import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
-import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -116,36 +111,6 @@ public record RoutingPolicy(RoutingPolicyId id,
return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints);
}
- /** Returns the zone endpoints of this */
- public List<Endpoint> zoneEndpointsIn(SystemName system, RoutingMethod routingMethod, boolean includeTokenEndpoint) {
- DeploymentId deployment = new DeploymentId(id.owner(), id.zone());
- Endpoint.EndpointBuilder builder = endpoint(routingMethod).target(id.cluster(), deployment);
- Endpoint zoneEndpoint = builder.in(system);
- List<Endpoint> endpoints = new ArrayList<>();
- endpoints.add(zoneEndpoint);
- if (includeTokenEndpoint) {
- Endpoint tokenEndpoint = builder.authMethod(Endpoint.AuthMethod.token).in(system);
- endpoints.add(tokenEndpoint);
- }
- for (var generatedEndpoint : generatedEndpoints) {
- boolean include = switch (generatedEndpoint.authMethod()) {
- case token -> includeTokenEndpoint;
- case mtls -> true;
- };
- if (include) {
- endpoints.add(builder.generatedFrom(generatedEndpoint).in(system));
- }
- }
- return endpoints;
- }
-
- /** Returns the region endpoint of this */
- public Endpoint regionEndpointIn(SystemName system, RoutingMethod routingMethod, Optional<GeneratedEndpoint> generated) {
- Endpoint.EndpointBuilder builder = endpoint(routingMethod).targetRegion(id.cluster(), id.zone());
- generated.ifPresent(builder::generatedFrom);
- return builder.in(system);
- }
-
@Override
public boolean equals(Object o) {
if (this == o) return true;
@@ -159,10 +124,4 @@ public record RoutingPolicy(RoutingPolicyId id,
return Objects.hash(id);
}
- private Endpoint.EndpointBuilder endpoint(RoutingMethod routingMethod) {
- return Endpoint.of(id.owner())
- .on(Port.fromRoutingMethod(routingMethod))
- .routingMethod(routingMethod);
- }
-
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
index 2e11a156dce..64a969a9c9d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
@@ -8,10 +8,12 @@ import com.yahoo.vespa.hosted.controller.LockedApplication;
import com.yahoo.vespa.hosted.controller.RoutingController;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.EndpointStatus;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
-import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
+import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml;
+import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpoints;
+import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
@@ -20,22 +22,21 @@ import java.time.Clock;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
-import java.util.Set;
/**
- * A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment.
+ * A deployment routing context. This extends {@link RoutingContext} to support configuration of routing for a deployment.
*
* @author mpolden
*/
public abstract class DeploymentRoutingContext implements RoutingContext {
final DeploymentId deployment;
- final RoutingController controller;
+ final RoutingController routing;
final RoutingMethod method;
- public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController controller) {
+ public DeploymentRoutingContext(DeploymentId deployment, RoutingMethod method, RoutingController routing) {
this.deployment = Objects.requireNonNull(deployment);
- this.controller = Objects.requireNonNull(controller);
+ this.routing = Objects.requireNonNull(routing);
this.method = Objects.requireNonNull(method);
}
@@ -44,13 +45,13 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
*
* @return the container endpoints relevant for this deployment, as declared in deployment spec
*/
- public final Set<ContainerEndpoint> prepare(LockedApplication application) {
- return controller.containerEndpointsOf(application, deployment.applicationId().instance(), deployment.zoneId());
+ public final PreparedEndpoints prepare(BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) {
+ return routing.prepare(deployment, services, certificate, application);
}
- /** Configure routing for the deployment in this context, using given deployment spec */
- public final void configure(DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) {
- controller.policies().refresh(deployment, deploymentSpec, generatedEndpoints);
+ /** Finalize routing configuration for the deployment in this context, using given deployment spec */
+ public final void activate(DeploymentSpec deploymentSpec, GeneratedEndpoints generatedEndpoints) {
+ routing.policies().refresh(deployment, deploymentSpec, generatedEndpoints);
}
/** Routing method of this context */
@@ -61,7 +62,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
/** Read the routing policy for given cluster in this deployment */
public final Optional<RoutingPolicy> routingPolicy(ClusterSpec.Id cluster) {
RoutingPolicyId id = new RoutingPolicyId(deployment.applicationId(), cluster, deployment.zoneId());
- return controller.policies().read(deployment).of(id);
+ return routing.policies().read(deployment).of(id);
}
/** Extension of a {@link DeploymentRoutingContext} for deployments using {@link RoutingMethod#sharedLayer4} routing */
@@ -110,13 +111,13 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
}
private List<String> upstreamNames() {
- List<String> upstreamNames = controller.readEndpointsOf(deployment)
- .scope(Endpoint.Scope.zone)
- .shared()
- .asList().stream()
- .map(endpoint -> endpoint.upstreamName(deployment))
- .distinct()
- .toList();
+ List<String> upstreamNames = routing.readEndpointsOf(deployment)
+ .scope(Endpoint.Scope.zone)
+ .shared()
+ .asList().stream()
+ .map(endpoint -> endpoint.upstreamName(deployment))
+ .distinct()
+ .toList();
if (upstreamNames.isEmpty()) {
throw new IllegalArgumentException("No upstream names found for " + deployment);
}
@@ -137,17 +138,17 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
@Override
public void setRoutingStatus(RoutingStatus.Value value, RoutingStatus.Agent agent) {
- controller.policies().setRoutingStatus(deployment, value, agent);
+ routing.policies().setRoutingStatus(deployment, value, agent);
}
@Override
public RoutingStatus routingStatus() {
// Status for a deployment applies to all clusters within the deployment, so we use the status from the
// first matching policy here
- return controller.policies().read(deployment)
- .first()
- .map(RoutingPolicy::routingStatus)
- .orElse(RoutingStatus.DEFAULT);
+ return routing.policies().read(deployment)
+ .first()
+ .map(RoutingPolicy::routingStatus)
+ .orElse(RoutingStatus.DEFAULT);
}
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index c46a28c4567..1ac811f0b4f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -16,11 +16,13 @@ import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.TenantName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.path.Path;
import com.yahoo.vespa.flags.PermanentFlags;
import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentData;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeploymentEndpoints;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.billing.PlanRegistryMock;
import com.yahoo.vespa.hosted.controller.api.integration.billing.Quota;
@@ -332,7 +334,8 @@ public class ControllerTest {
List.of("beta.app1.tenant1.global.vespa.oath.cloud",
"rotation-id-01"),
OptionalInt.empty(),
- RoutingMethod.sharedLayer4));
+ RoutingMethod.sharedLayer4,
+ Map.of("beta.app1.tenant1.global.vespa.oath.cloud", AuthMethod.mtls)));
for (Deployment deployment : betaDeployments) {
assertEquals(containerEndpoints,
@@ -350,7 +353,8 @@ public class ControllerTest {
List.of("app1.tenant1.global.vespa.oath.cloud",
"rotation-id-02"),
OptionalInt.empty(),
- RoutingMethod.sharedLayer4));
+ RoutingMethod.sharedLayer4,
+ Map.of("app1.tenant1.global.vespa.oath.cloud", AuthMethod.mtls)));
for (Deployment deployment : defaultDeployments) {
assertEquals(containerEndpoints,
tester.configServer().containerEndpoints().get(defaultContext.deploymentIdIn(deployment.zone())));
@@ -740,10 +744,14 @@ public class ControllerTest {
);
deploymentEndpoints.forEach((deployment, endpoints) -> {
Set<ContainerEndpoint> expected = endpoints.entrySet().stream()
- .map(kv -> new ContainerEndpoint("default", "application",
+ .map(kv -> {
+ Map<String, AuthMethod> authMethods = kv.getKey().stream().collect(Collectors.toMap(Function.identity(), (v) -> AuthMethod.mtls));
+ return new ContainerEndpoint("default", "application",
kv.getKey(),
OptionalInt.of(kv.getValue()),
- tester.controller().zoneRegistry().routingMethod(deployment.zoneId())))
+ tester.controller().zoneRegistry().routingMethod(deployment.zoneId()),
+ authMethods);
+ })
.collect(Collectors.toSet());
assertEquals(expected,
tester.configServer().containerEndpoints().get(deployment),
@@ -790,7 +798,7 @@ public class ControllerTest {
RecordName.from("e.app1.tenant1.a.vespa.oath.cloud"),
RecordData.from("vip.prod.us-east-3.")))),
new TreeSet<>(records));
- List<String> endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application())
+ List<String> endpointDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.application())
.scope(Endpoint.Scope.application)
.sortedBy(comparing(Endpoint::dnsName))
.mapToList(Endpoint::dnsName);
@@ -1536,8 +1544,8 @@ public class ControllerTest {
DeploymentContext context = tester.newDeploymentContext();
DeploymentId deployment = context.deploymentIdIn(ZoneId.from("prod", "us-west-1"));
DeploymentData deploymentData = new DeploymentData(deployment.applicationId(), deployment.zoneId(), InputStream::nullInputStream, Version.fromString("6.1"),
- Set.of(), Optional::empty, Optional.empty(), Optional.empty(),
- Quota::unlimited, List.of(), List.of(), Optional::empty, List.of(),false);
+ () -> DeploymentEndpoints.none, Optional.empty(), Optional.empty(),
+ Quota::unlimited, List.of(), List.of(), Optional::empty, List.of(), false);
tester.configServer().deploy(deploymentData);
assertTrue(tester.configServer().application(deployment.applicationId(), deployment.zoneId()).isPresent());
tester.controller().applications().deactivate(deployment.applicationId(), deployment.zoneId());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
index 477aca86b9c..cc7a001b0b4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
@@ -269,7 +270,7 @@ public class EndpointTest {
Endpoint.of(instance1)
.targetRegion(ClusterSpec.Id.from("c1"), prodZone)
.routingMethod(RoutingMethod.exclusive)
- .generatedFrom(new GeneratedEndpoint("deadbeef", "cafed00d", Endpoint.AuthMethod.mtls))
+ .generatedFrom(new GeneratedEndpoint("deadbeef", "cafed00d", AuthMethod.mtls))
.on(Port.tls())
.in(SystemName.Public)
);
@@ -353,7 +354,7 @@ public class EndpointTest {
@Test
public void generated_id() {
- GeneratedEndpoint ge = new GeneratedEndpoint("cafed00d", "deadbeef", Endpoint.AuthMethod.mtls);
+ GeneratedEndpoint ge = new GeneratedEndpoint("cafed00d", "deadbeef", AuthMethod.mtls);
var deployment = new DeploymentId(instance1, ZoneId.from("prod", "us-north-1"));
var tests = Map.of(
// Zone endpoint in main, unlike named endpoints, this includes the scope symbol 'z'
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java
new file mode 100644
index 00000000000..7d377ef6361
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/pkg/BasicServicesXmlTest.java
@@ -0,0 +1,54 @@
+package com.yahoo.vespa.hosted.controller.application.pkg;
+
+import com.yahoo.text.XML;
+import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml.Container;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * @author mpolden
+ */
+class BasicServicesXmlTest {
+
+ @Test
+ public void parse() {
+ assertServices(new BasicServicesXml(List.of()), "<services/>");
+ assertServices(new BasicServicesXml(List.of(new Container("foo", List.of(Container.AuthMethod.mtls)),
+ new Container("bar", List.of(Container.AuthMethod.mtls)))),
+ """
+ <services>
+ <container id="foo"/>
+ <container id="bar"/>
+ </services>
+ """);
+ assertServices(new BasicServicesXml(List.of(
+ new Container("foo",
+ List.of(Container.AuthMethod.mtls,
+ Container.AuthMethod.token)),
+ new Container("bar", List.of(Container.AuthMethod.mtls)))),
+ """
+ <services>
+ <container id="foo">
+ <clients>
+ <client id="mtls"/>
+ <client id="token">
+ <token id="my-token"/>
+ </client>
+ <client id="token2">
+ <token id="other-token"/>
+ </client>
+ </clients>
+ </container>
+ <container id="bar"/>
+ </services>
+ """);
+ }
+
+ private void assertServices(BasicServicesXml expected, String xmlForm) {
+ assertEquals(expected, BasicServicesXml.parse(XML.getDocument(xmlForm)));
+ }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index fb3026e1d80..f417e3d52fb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -8,6 +8,7 @@ import com.yahoo.config.provision.AthenzService;
import com.yahoo.config.provision.Environment;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.security.SignatureAlgorithm;
import com.yahoo.security.X509CertificateBuilder;
import com.yahoo.security.X509CertificateUtils;
@@ -56,6 +57,7 @@ public class ApplicationPackageBuilder {
"/>\n</notifications>\n").setEmptyValue("");
private final StringBuilder endpointsBody = new StringBuilder();
private final StringBuilder applicationEndpointsBody = new StringBuilder();
+ private final StringBuilder servicesBody = new StringBuilder();
private final List<X509Certificate> trustedCertificates = new ArrayList<>();
private final Map<Environment, Map<String, String>> nonProductionEnvironments = new LinkedHashMap<>();
@@ -112,6 +114,28 @@ public class ApplicationPackageBuilder {
return this;
}
+ public ApplicationPackageBuilder container(String id, AuthMethod... authMethod) {
+ servicesBody.append(" <container id='")
+ .append(id)
+ .append("'>\n")
+ .append(" <clients>\n");
+ for (int i = 0; i < authMethod.length; i++) {
+ AuthMethod m = authMethod[i];
+ servicesBody.append(" <client id='")
+ .append("client-").append(m.name()).append("-").append(i)
+ .append("'>\n");
+ if (m == AuthMethod.token) {
+ servicesBody.append(" <token id='")
+ .append(m.name()).append("-").append(i)
+ .append("'/>\n");
+ }
+ servicesBody.append(" </client>\n");
+ }
+ servicesBody.append(" </clients>\n")
+ .append(" </container>\n");
+ return this;
+ }
+
public ApplicationPackageBuilder applicationEndpoint(String id, String containerId, String region,
Map<InstanceName, Integer> instanceWeights) {
return applicationEndpoint(id, containerId, Map.of(region, instanceWeights));
@@ -350,6 +374,10 @@ public class ApplicationPackageBuilder {
return searchDefinition.getBytes(UTF_8);
}
+ private byte[] services() {
+ return ("<services version='1.0'>\n" + servicesBody + "</services>\n").getBytes(UTF_8);
+ }
+
private static byte[] buildMeta(Version compileVersion) {
return compileVersion == null ? new byte[0]
: ("{\"compileVersion\":\"" + compileVersion.toFullString() +
@@ -362,6 +390,7 @@ public class ApplicationPackageBuilder {
try (ZipOutputStream out = new ZipOutputStream(zip)) {
out.setLevel(Deflater.NO_COMPRESSION); // This is for testing purposes so we skip compression for performance
writeZipEntry(out, "deployment.xml", deploymentSpec());
+ writeZipEntry(out, "services.xml", services());
writeZipEntry(out, "validation-overrides.xml", validationOverrides());
writeZipEntry(out, "schemas/test.sd", searchDefinition());
writeZipEntry(out, "build-meta.json", buildMeta(compileVersion));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 3fb4a040e0d..62bdf95515d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -29,6 +29,7 @@ import com.yahoo.vespa.hosted.controller.integration.ZoneRegistryMock;
import com.yahoo.vespa.hosted.controller.maintenance.DeploymentUpgrader;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import com.yahoo.vespa.hosted.controller.versions.VespaVersion.Confidence;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import java.time.Duration;
@@ -2912,6 +2913,7 @@ public class DeploymentTriggerTest {
}
@Test
+ @Disabled // For benchmarking, not a test
void miniBenchmark() {
String spec = """
<deployment version="1.0">
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 0862496275a..a03583c4a59 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -409,12 +409,12 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
applications.put(id, new Application(id.applicationId(), lastPrepareVersion, appPackage));
ClusterSpec.Id cluster = ClusterSpec.Id.from("default");
- deployment.endpointCertificate(); // Supplier with side effects >_<
+ deployment.endpoints(); // Supplier with side effects >_<
if (nodeRepository().list(id.zoneId(), NodeFilter.all().applications(id.applicationId())).isEmpty())
provision(id.zoneId(), id.applicationId(), cluster);
- this.containerEndpoints.put(id, deployment.containerEndpoints());
+ this.containerEndpoints.put(id, deployment.endpoints().get().endpoints());
deployment.cloudAccount().ifPresent(account -> this.cloudAccounts.put(id, account));
if (!deferLoadBalancerProvisioning.contains(id.zoneId().environment())) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
index f685c75bbe3..d9007910541 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
@@ -4,8 +4,8 @@ package com.yahoo.vespa.hosted.controller.persistence;
import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
@@ -47,7 +47,7 @@ public class RoutingPolicySerializerTest {
Set.of(),
RoutingStatus.DEFAULT,
false,
- List.of(new GeneratedEndpoint("deadbeef", "cafed00d", Endpoint.AuthMethod.mtls))),
+ List.of(new GeneratedEndpoint("deadbeef", "cafed00d", AuthMethod.mtls))),
new RoutingPolicy(id2,
Optional.of(HostName.of("long-and-ugly-name-2")),
Optional.empty(),
@@ -58,7 +58,7 @@ public class RoutingPolicySerializerTest {
RoutingStatus.Agent.tenant,
Instant.ofEpochSecond(123)),
true,
- List.of(new GeneratedEndpoint("cafed00d", "deadbeef", Endpoint.AuthMethod.token))),
+ List.of(new GeneratedEndpoint("cafed00d", "deadbeef", AuthMethod.token))),
new RoutingPolicy(id1,
Optional.empty(),
Optional.of("127.0.0.1"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 46ec42cab8f..1dfaf2109c7 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -14,6 +14,7 @@ import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.InstanceName;
import com.yahoo.config.provision.RegionName;
import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.AuthMethod;
import com.yahoo.config.provision.zone.RoutingMethod;
import com.yahoo.config.provision.zone.ZoneId;
import com.yahoo.vespa.flags.Flags;
@@ -41,6 +42,7 @@ import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue;
import com.yahoo.vespa.hosted.controller.dns.RemoveRecords;
import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
@@ -334,6 +336,10 @@ public class RoutingPoliciesTest {
var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
// Deploy application
+ ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
+ .region(zone2.region())
+ .container("c0", AuthMethod.mtls, AuthMethod.token)
+ .build();
tester.provisionLoadBalancers(1, context1.instanceId(), false, zone1, zone2);
context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
@@ -358,12 +364,23 @@ public class RoutingPoliciesTest {
// Ordinary endpoints are not created in DNS
assertEquals(List.of(), tester.recordNames());
assertEquals(2, tester.policiesOf(context.instanceId()).size());
- // Generated endpoints are created in DNS
+ }
+
+ @Test
+ @Disabled // TODO(mpolden): Enable this test when we start creating generated endpoints for shared routing
+ void zone_routing_policies_with_shared_routing_and_generated_endpoint() {
+ var tester = new RoutingPoliciesTester(new DeploymentTester(), false);
+ var context = tester.newDeploymentContext("tenant1", "app1", "default");
+ tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2);
tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester);
+ ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
+ .region(zone2.region())
+ .container("c0", AuthMethod.mtls, AuthMethod.token)
+ .build();
context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
- assertEquals(List.of("b22ab332.cafed00d.z.vespa.oath.cloud",
- "d71005bf.cafed00d.z.vespa.oath.cloud"),
+ assertEquals(List.of("c0a25b7c.cafed00d.z.vespa.oath.cloud",
+ "dc5e383c.cafed00d.z.vespa.oath.cloud"),
tester.recordNames());
}
@@ -757,7 +774,7 @@ public class RoutingPoliciesTest {
tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out,
RoutingStatus.Agent.tenant);
} catch (IllegalArgumentException e) {
- assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive, authMethod=mtls]", e.getMessage());
+ assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive, authMethod=mtls, name=r0]", e.getMessage());
}
context.flushDnsUpdates();
tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
@@ -942,7 +959,7 @@ public class RoutingPoliciesTest {
tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
fail("Expected exception");
} catch (IllegalArgumentException e) {
- assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive, authMethod=mtls]",
+ assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive, authMethod=mtls, name=a0]",
e.getMessage());
}
@@ -993,14 +1010,17 @@ public class RoutingPoliciesTest {
var tester = new RoutingPoliciesTester(SystemName.Public);
var context = tester.newDeploymentContext("tenant1", "app1", "default");
tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
+ tester.enableTokenEndpoint(true);
addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester);
// Deploy application
- int clustersPerZone = 1;
+ int clustersPerZone = 2;
var zone1 = ZoneId.from("prod", "aws-us-east-1c");
var zone2 = ZoneId.from("prod", "aws-eu-west-1a");
ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
.region(zone2.region())
+ .container("c0", AuthMethod.mtls)
+ .container("c1", AuthMethod.mtls, AuthMethod.token)
.endpoint("foo", "c0")
.applicationEndpoint("bar", "c0", Map.of(zone1.region().value(), Map.of(InstanceName.defaultName(), 1)))
.build();
@@ -1011,6 +1031,8 @@ public class RoutingPoliciesTest {
List<String> expectedRecords = List.of(
// save me, jebus!
"b22ab332.cafed00d.z.vespa-app.cloud",
+ "b7e79800.cafed00d.z.vespa-app.cloud",
+ "b8ee0967.cafed00d.z.vespa-app.cloud",
"bar.app1.tenant1.a.vespa-app.cloud",
"bar.cafed00d.a.vespa-app.cloud",
"c0.app1.tenant1.aws-eu-west-1.w.vespa-app.cloud",
@@ -1019,26 +1041,42 @@ public class RoutingPoliciesTest {
"c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
"c0.cafed00d.aws-eu-west-1.w.vespa-app.cloud",
"c0.cafed00d.aws-us-east-1.w.vespa-app.cloud",
- "dd0971b4.cafed00d.z.vespa-app.cloud",
+ "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+ "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+ "c60d3149.cafed00d.z.vespa-app.cloud",
+ "cbff1506.cafed00d.z.vespa-app.cloud",
+ "d151139b.cafed00d.z.vespa-app.cloud",
"foo.app1.tenant1.g.vespa-app.cloud",
- "foo.cafed00d.g.vespa-app.cloud"
+ "foo.cafed00d.g.vespa-app.cloud",
+ "token-c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+ "token-c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
- assertEquals(2, tester.policiesOf(context.instanceId()).size());
+ assertEquals(4, tester.policiesOf(context.instanceId()).size());
+ ClusterSpec.Id cluster0 = ClusterSpec.Id.from("c0");
+ ClusterSpec.Id cluster1 = ClusterSpec.Id.from("c1");
for (var zone : List.of(zone1, zone2)) {
- EndpointList endpoints = tester.controllerTester().controller().routing().readEndpointsOf(context.deploymentIdIn(zone)).scope(Endpoint.Scope.zone);
- assertEquals(1, endpoints.generated().size());
+ EndpointList generated = tester.controllerTester().controller().routing()
+ .readEndpointsOf(context.deploymentIdIn(zone))
+ .scope(Endpoint.Scope.zone)
+ .generated();
+ assertEquals(1, generated.cluster(cluster0).size());
+ assertEquals(0, generated.cluster(cluster0).authMethod(AuthMethod.token).size());
+ assertEquals(2, generated.cluster(cluster1).size());
+ assertEquals(1, generated.cluster(cluster1).authMethod(AuthMethod.token).size());
}
+
// Ordinary endpoints point to expected targets
- tester.assertTargets(context.instanceId(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0,
+ tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0,
Map.of(zone1, 1L, zone2, 1L));
- tester.assertTargets(context.application().id(), EndpointId.of("bar"), ClusterSpec.Id.from("c0"), 0,
+ tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0,
Map.of(context.deploymentIdIn(zone1), 1));
+
// Generated endpoints point to expected targets
- tester.assertTargets(context.instanceId(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0,
+ tester.assertTargets(context.instanceId(), EndpointId.of("foo"), cluster0, 0,
Map.of(zone1, 1L, zone2, 1L),
true);
- tester.assertTargets(context.application().id(), EndpointId.of("bar"), ClusterSpec.Id.from("c0"), 0,
+ tester.assertTargets(context.application().id(), EndpointId.of("bar"), cluster0, 0,
Map.of(context.deploymentIdIn(zone1), 1),
true);