diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-07-04 11:57:49 +0200 |
---|---|---|
committer | Martin Polden <mpolden@mpolden.no> | 2023-07-05 12:52:00 +0200 |
commit | d56134d93b7d62f5e96b688185d3d5b64d0bf942 (patch) | |
tree | cfb60fe3ef1a99288be9974d0eb65984a6f5d777 /controller-server | |
parent | 8fc733d22134a4645e6b3bb4cdde524cb251f5ae (diff) |
Support anonymized endpoints
Diffstat (limited to 'controller-server')
21 files changed, 475 insertions, 200 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index adadcab3270..fdb27ba49a3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -55,6 +55,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics; import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning; import com.yahoo.vespa.hosted.controller.application.DeploymentQuotaCalculator; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.QuotaUsage; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; @@ -88,6 +89,7 @@ import java.security.cert.X509Certificate; import java.time.Clock; import java.time.Duration; import java.time.Instant; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -150,6 +152,7 @@ public class ApplicationController { private final ListFlag<String> incompatibleVersions; private final BillingController billingController; private final ListFlag<String> cloudAccountsFlag; + private final Map<DeploymentId, com.yahoo.vespa.hosted.controller.api.integration.configserver.Application> deploymentInfo = new ConcurrentHashMap<>(); ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock, @@ -651,7 +654,8 @@ public class ApplicationController { DeploymentId deployment = new DeploymentId(application, zone); // Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails. interface CleanCloseable extends AutoCloseable { void close(); } - try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage)) { + List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>(); + try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage, generatedEndpoints)) { Optional<DockerImage> dockerImageRepo = Optional.ofNullable( dockerImageRepoFlag .with(FetchVector.Dimension.ZONE_ID, zone.value()) @@ -680,8 +684,16 @@ public class ApplicationController { } Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec()); List<DataplaneTokenVersions> dataplaneTokenVersions = controller.dataplaneTokenService().listTokens(application.tenant()); + Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadataWrapper = () -> { + Optional<EndpointCertificateMetadata> data = endpointCertificateMetadata.get(); + // TODO(mpolden): Pass these endpoints to config server as part of the deploy call. This will let the + // application know which endpoints are mTLS and which are token-based + data.flatMap(EndpointCertificateMetadata::randomizedId) + .ifPresent(applicationPart -> generatedEndpoints.addAll(controller.routing().generateEndpoints(applicationPart, deployment.applicationId()))); + return data; + }; DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform, - endpoints, endpointCertificateMetadata, dockerImageRepo, domain, + endpoints, endpointCertificateMetadataWrapper, dockerImageRepo, domain, deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun); ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData); @@ -689,9 +701,9 @@ public class ApplicationController { } } - private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data) { + private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, List<GeneratedEndpoint> generatedEndpoints) { if (id.applicationId().instance().isTester()) return; - controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec()); + controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec(), generatedEndpoints); if ( ! id.zoneId().environment().isManuallyDeployed()) return; controller.applications().applicationStore().putMeta(id, clock.instant(), data.truncatedPackage().metaDataZip()); } @@ -906,7 +918,7 @@ public class ApplicationController { DeploymentId id = new DeploymentId(instanceId, zone); interface CleanCloseable extends AutoCloseable { void close(); } try (CleanCloseable postDeactivation = () -> { - application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec())); + application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec(), List.of())); if (id.zoneId().environment().isManuallyDeployed()) applicationStore.putMetaTombstone(id, clock.instant()); if ( ! id.zoneId().environment().isTest()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 287cfaa41b8..f6bcbc9828b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -27,8 +27,8 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationsDb; import com.yahoo.vespa.hosted.controller.notification.Notifier; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags; -import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService; +import com.yahoo.vespa.hosted.controller.security.AccessControl; import com.yahoo.vespa.hosted.controller.support.access.SupportAccessControl; import com.yahoo.vespa.hosted.controller.versions.OsVersion; import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus; @@ -38,12 +38,14 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.yolean.concurrent.Sleeper; +import java.security.SecureRandom; import java.time.Clock; import java.time.Instant; import java.util.LinkedHashMap; import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.TreeSet; import java.util.function.Function; @@ -91,6 +93,8 @@ public class Controller extends AbstractComponent { private final Notifier notifier; private final MailVerifier mailVerifier; private final DataplaneTokenService dataplaneTokenService; + private final Random random; + private final Random secureRandom; // Type is Random to allow for test determinism /** * Creates a controller @@ -102,13 +106,14 @@ public class Controller extends AbstractComponent { MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, ControllerConfig controllerConfig) { this(curator, rotationsConfig, accessControl, flagSource, - mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT); + mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT, new Random(), + new SecureRandom()); } public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl, FlagSource flagSource, MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore, - ControllerConfig controllerConfig, Sleeper sleeper) { + ControllerConfig controllerConfig, Sleeper sleeper, Random random, Random secureRandom) { this.curator = Objects.requireNonNull(curator, "Curator cannot be null"); this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null"); this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null"); @@ -119,6 +124,8 @@ public class Controller extends AbstractComponent { this.metric = Objects.requireNonNull(metric, "Metric cannot be null"); this.controllerConfig = Objects.requireNonNull(controllerConfig, "ControllerConfig cannot be null"); this.secretStore = Objects.requireNonNull(secretStore, "SecretStore cannot be null"); + this.random = Objects.requireNonNull(random, "Random cannot be null"); + this.secureRandom = Objects.requireNonNull(secureRandom, "SecureRandom cannot be null"); nameServiceForwarder = new NameServiceForwarder(curator); jobController = new JobController(this); @@ -362,4 +369,11 @@ public class Controller extends AbstractComponent { public DataplaneTokenService dataplaneTokenService() { return dataplaneTokenService; } + + /** Returns a random number generator. If secure is true, this returns a {@link SecureRandom} suitable for + * cryptographic purposes */ + public Random random(boolean secure) { + return secure ? secureRandom : random; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index d0b34b6094d..ceac681255b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.Endpoint.Scope; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; @@ -44,6 +45,7 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Comparator; @@ -75,6 +77,7 @@ public class RoutingController { private final RoutingPolicies routingPolicies; private final RotationRepository rotationRepository; private final BooleanFlag createTokenEndpoint; + private final BooleanFlag randomizedEndpoints; public RoutingController(Controller controller, RotationsConfig rotationsConfig) { this.controller = Objects.requireNonNull(controller, "controller must be non-null"); @@ -83,6 +86,7 @@ public class RoutingController { controller.applications(), controller.curator()); this.createTokenEndpoint = Flags.ENABLE_DATAPLANE_PROXY.bindTo(controller.flagSource()); + this.randomizedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource()); } /** Create a routing context for given deployment */ @@ -138,6 +142,7 @@ public class RoutingController { /** Returns endpoints declared in {@link DeploymentSpec} for given application */ public EndpointList declaredEndpointsOf(Application application) { + // TODO(mpolden): Add generated endpoints for global and application scopes. Requires reading routing polices here Set<Endpoint> endpoints = new LinkedHashSet<>(); DeploymentSpec deploymentSpec = application.deploymentSpec(); for (var spec : deploymentSpec.instances()) { @@ -169,7 +174,6 @@ public class RoutingController { t -> t.weight())); ZoneId zone = deployments.keySet().iterator().next().zoneId(); // Where multiple zones are possible, they all have the same routing method. - // Application endpoints are only supported when using direct routing methods RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive; endpoints.add(Endpoint.of(application.id()) .targetApplication(EndpointId.of(declaredEndpoint.endpointId()), @@ -358,6 +362,19 @@ public class RoutingController { Optional.of(application.id()))); } + /** Generate endpoints for all authenticaiton methods, using given application part */ + public List<GeneratedEndpoint> generateEndpoints(String applicationPart, ApplicationId instance) { + boolean enabled = randomizedEndpoints.with(FetchVector.Dimension.APPLICATION_ID, instance.serializedForm()).value(); + if (!enabled) { + return List.of(); + } + return Arrays.stream(Endpoint.AuthMethod.values()) + .map(method -> new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), + applicationPart, + method)) + .toList(); + } + /** * Assigns one or more global rotations to given application, if eligible. The given application is implicitly * stored, ensuring that the assigned rotation(s) are persisted when this returns. diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index f55bb0dab6f..a3381819778 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -44,11 +44,12 @@ public class Endpoint { private final Scope scope; private final boolean legacy; private final RoutingMethod routingMethod; - private boolean tokenEndpoint; + private final AuthMethod authMethod; + private final boolean generated; private Endpoint(TenantAndApplicationId application, Optional<InstanceName> instanceName, EndpointId id, ClusterSpec.Id cluster, URI url, List<Target> targets, Scope scope, Port port, boolean legacy, - RoutingMethod routingMethod, boolean certificateName, boolean tokenEndpoint) { + RoutingMethod routingMethod, boolean certificateName, AuthMethod authMethod, boolean generated) { Objects.requireNonNull(application, "application must be non-null"); Objects.requireNonNull(instanceName, "instanceName must be non-null"); Objects.requireNonNull(cluster, "cluster must be non-null"); @@ -57,6 +58,7 @@ public class Endpoint { Objects.requireNonNull(scope, "scope must be non-null"); Objects.requireNonNull(port, "port must be non-null"); Objects.requireNonNull(routingMethod, "routingMethod must be non-null"); + Objects.requireNonNull(authMethod, "authMethod must be non-null"); this.id = requireEndpointId(id, scope, certificateName); this.cluster = requireCluster(cluster, certificateName); this.instance = requireInstance(instanceName, scope); @@ -65,7 +67,8 @@ public class Endpoint { this.scope = requireScope(scope, routingMethod); this.legacy = legacy; this.routingMethod = routingMethod; - this.tokenEndpoint = tokenEndpoint; + this.authMethod = authMethod; + this.generated = generated; } /** @@ -135,6 +138,11 @@ public class Endpoint { return routingMethod.isShared() && scope == Scope.global; } + /** Returns whether this endpoint is generated by the system */ + public boolean generated() { + return generated; + } + /** Returns the upstream name of given deployment. This *must* match what the routing layer generates */ public String upstreamName(DeploymentId deployment) { if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name"); @@ -156,7 +164,7 @@ public class Endpoint { @Override public String toString() { - return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s]", url, scope, legacy, routingMethod); + return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s]", url, scope, legacy, routingMethod, authMethod); } private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) { @@ -164,24 +172,39 @@ public class Endpoint { } private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance, - List<Target> targets, Scope scope, SystemName system, Port port, boolean legacyRegionalUrl) { + List<Target> targets, Scope scope, SystemName system, Port port, + Optional<GeneratedEndpoint> generated) { String separator = "."; String portPart = port.isDefault() ? "" : ":" + port.port; + final String subdomain; + if (generated.isPresent()) { + subdomain = generatedPart(generated.get(), name, scope, separator); + } else { + subdomain = sanitize(namePart(name, separator)) + + systemPart(system, separator) + + sanitize(instancePart(instance, separator)) + + sanitize(application.application().value()) + + separator + + sanitize(application.tenant().value()); + } return URI.create("https://" + - sanitize(namePart(name, separator)) + - systemPart(system, separator) + - sanitize(instancePart(instance, separator)) + - sanitize(application.application().value()) + - separator + - sanitize(application.tenant().value()) + + subdomain + "." + - scopePart(scope, targets, system, legacyRegionalUrl) + + scopePart(scope, targets, system, generated) + dnsSuffix(system) + portPart + "/"); } + private static String generatedPart(GeneratedEndpoint generated, String name, Scope scope, String separator) { + if (scope.multiDeployment()) { + // Endpoints with these scopes have a name part that is explicitly configured through deployment.xml + return sanitize(namePart(name, separator)) + generated.applicationPart(); + } + return generated.clusterPart() + separator + generated.applicationPart(); + } + private static String sanitize(String part) { // TODO: Reject reserved words return part.replace('_', '-'); } @@ -191,24 +214,23 @@ public class Endpoint { return name + separator; } - private static String scopePart(Scope scope, List<Target> targets, SystemName system, boolean legacyRegion) { - String scopeSymbol = scopeSymbol(scope, system, legacyRegion); + private static String scopePart(Scope scope, List<Target> targets, SystemName system, Optional<GeneratedEndpoint> generated) { + String scopeSymbol = scopeSymbol(scope, system, generated); if (scope == Scope.global) return scopeSymbol; - if (scope == Scope.application && ! legacyRegion) return scopeSymbol; + if (scope == Scope.application) return scopeSymbol; + if (generated.isPresent()) return scopeSymbol; ZoneId zone = targets.stream().map(target -> target.deployment.zoneId()).min(comparing(ZoneId::value)).get(); String region = zone.region().value(); - boolean skipEnvironment = zone.environment().isProduction(); - String environment = skipEnvironment ? "" : "." + zone.environment().value(); + String environment = zone.environment().isProduction() ? "" : "." + zone.environment().value(); if (system.isPublic()) { return region + environment + "." + scopeSymbol; } return region + (scopeSymbol.isEmpty() ? "" : "-" + scopeSymbol) + environment; } - private static String scopeSymbol(Scope scope, SystemName system, boolean legacyRegion) { - if (legacyRegion) return "r"; - if (system.isPublic()) { + private static String scopeSymbol(Scope scope, SystemName system, Optional<GeneratedEndpoint> generated) { + if (system.isPublic() || generated.isPresent()) { return switch (scope) { case zone -> "z"; case weighted -> "w"; @@ -347,8 +369,9 @@ public class Endpoint { return targets; } - public boolean isTokenEndpoint() { - return tokenEndpoint; + /** Returns the authentication method of this endpoint */ + public AuthMethod authMethod() { + return authMethod; } /** An endpoint's scope */ @@ -381,22 +404,25 @@ public class Endpoint { } + /** An endpoint's authentication method */ + public enum AuthMethod { + mtls, + token, + } + /** Represents an endpoint's HTTP port */ - public static class Port { + public record Port(int port) { private static final Port TLS_DEFAULT = new Port(443); - private final int port; - - private Port(int port) { + public Port { if (port < 1 || port > 65535) { throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port); } - this.port = port; } private boolean isDefault() { - return port == 443; + return port == TLS_DEFAULT.port; } /** Returns the default HTTPS port */ @@ -407,12 +433,7 @@ public class Endpoint { /** Returns default port for the given routing method */ public static Port fromRoutingMethod(RoutingMethod method) { if (method.isDirect()) return Port.tls(); - return Port.tls(4443); - } - - /** Create a HTTPS port */ - public static Port tls(int port) { - return new Port(port); + return new Port(4443); } } @@ -475,14 +496,15 @@ public class Endpoint { private RoutingMethod routingMethod = RoutingMethod.sharedLayer4; private boolean legacy = false; private boolean certificateName = false; - private boolean tokenEndpoint = false; + private AuthMethod authMethod = AuthMethod.mtls; + private Optional<GeneratedEndpoint> generated = Optional.empty(); private EndpointBuilder(TenantAndApplicationId application, Optional<InstanceName> instance) { this.application = Objects.requireNonNull(application); this.instance = Objects.requireNonNull(instance); } - /** Sets the deployment target for this */ + /** Sets the zone target for this */ public EndpointBuilder target(ClusterSpec.Id cluster, DeploymentId deployment) { this.cluster = cluster; this.scope = requireUnset(Scope.zone); @@ -543,8 +565,9 @@ public class Endpoint { return this; } - public EndpointBuilder tokenEndpoint() { - this.tokenEndpoint = true; + /** Sets the valid authentication method supported by this */ + public EndpointBuilder authMethod(AuthMethod authMethod) { + this.authMethod = authMethod; return this; } @@ -572,23 +595,32 @@ public class Endpoint { return this; } + /** Sets the generated ID to use when building this */ + public EndpointBuilder generatedEndpoint(GeneratedEndpoint generated) { + this.generated = Optional.of(generated); + this.authMethod = generated.authMethod(); + return this; + } + /** Sets the system that owns this */ public Endpoint in(SystemName system) { if (system.isPublic() && routingMethod != RoutingMethod.exclusive) { throw new IllegalArgumentException("Public system only supports routing method " + RoutingMethod.exclusive); } - if (routingMethod.isDirect() && !port.isDefault()) { - throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port"); - } - String prefix = tokenEndpoint ? "token-" : ""; - URI url = createUrl(prefix + endpointOrClusterAsString(endpointId, cluster), + String prefix = authMethod == AuthMethod.token ? "token-" : ""; + String name = endpointOrClusterAsString(endpointId, Objects.requireNonNull(cluster, "cluster must be non-null")); + URI url = createUrl(prefix + name, Objects.requireNonNull(application, "application must be non-null"), Objects.requireNonNull(instance, "instance must be non-null"), Objects.requireNonNull(targets, "targets must be non-null"), Objects.requireNonNull(scope, "scope must be non-null"), Objects.requireNonNull(system, "system must be non-null"), Objects.requireNonNull(port, "port must be non-null"), - false); + Objects.requireNonNull(generated) + ); + if (routingMethod.isDirect() && !port.isDefault()) { + throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port"); + } return new Endpoint(application, instance, endpointId, @@ -600,7 +632,8 @@ public class Endpoint { legacy, routingMethod, certificateName, - tokenEndpoint); + authMethod, + generated.isPresent()); } private Scope requireUnset(Scope scope) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java index de1429d88af..b7ca8587efa 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java @@ -4,8 +4,7 @@ package com.yahoo.vespa.hosted.controller.application; import java.util.Objects; /** - * A type to represent the ID of an endpoint. This is typically the first part of - * an endpoint name. + * A user-specified endpoint ID. This is typically the first part of an endpoint name. * * @author ogronnesby */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java index 3da9065b52d..5026fea7847 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java @@ -64,6 +64,11 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return matching(Endpoint::legacy); } + /** Returns the subset of endpoints generated by the system */ + public EndpointList generated() { + return matching(Endpoint::generated); + } + /** Returns the subset of endpoints that require a rotation */ public EndpointList requiresRotation() { return matching(Endpoint::requiresRotation); @@ -88,4 +93,9 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList> return new EndpointList(endpoints, false); } + @Override + public String toString() { + return asList().toString(); + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java new file mode 100644 index 00000000000..dd6f4e5111d --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java @@ -0,0 +1,34 @@ +package com.yahoo.vespa.hosted.controller.application; + +import ai.vespa.validation.Validation; + +import java.util.random.RandomGenerator; +import java.util.regex.Pattern; + +/** + * A system-generated endpoint, where the cluster and application parts are randomly generated. These become the + * first and second part of an endpoint name. See {@link Endpoint}. + * + * @author mpolden + */ +public record GeneratedEndpoint(String clusterPart, String applicationPart, Endpoint.AuthMethod authMethod) { + + private static final Pattern PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$"); + + public GeneratedEndpoint { + Validation.requireMatch(clusterPart, "Cluster part", PART_PATTERN); + Validation.requireMatch(applicationPart, "Application part", PART_PATTERN); + } + + /** Create a new endpoint part, using random as a source of randomness */ + public static String createPart(RandomGenerator random) { + String alphabet = "abcdef0123456789"; + StringBuilder sb = new StringBuilder(); + sb.append(alphabet.charAt(random.nextInt(6))); // Start with letter + for (int i = 0; i < 7; i++) { + sb.append(alphabet.charAt(random.nextInt(alphabet.length()))); + } + return sb.toString(); + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index 82d4f068d6d..46d19b627cc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -14,9 +14,10 @@ import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; +import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import java.time.Duration; @@ -27,7 +28,6 @@ import java.util.OptionalInt; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -import java.util.random.RandomGenerator; import java.util.stream.Collectors; /** @@ -39,7 +39,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { private static final Logger log = Logger.getLogger(CertificatePoolMaintainer.class.getName()); - private final RandomGenerator random; private final CuratorDb curator; private final SecretStore secretStore; private final EndpointCertificateProvider endpointCertificateProvider; @@ -50,7 +49,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; - public CertificatePoolMaintainer(Controller controller, Metric metric, Duration interval, RandomGenerator random) { + public CertificatePoolMaintainer(Controller controller, Metric metric, Duration interval) { super(controller, interval, null, Set.of(SystemName.Public, SystemName.PublicCd)); this.controller = controller; this.secretStore = controller.secretStore(); @@ -61,7 +60,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); this.metric = metric; this.dnsSuffix = Endpoint.dnsSuffix(controller.system()); - this.random = random; } protected double maintain() { @@ -129,12 +127,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { } private String generateRandomId() { - String alphabet = "abcdef0123456789"; - StringBuilder sb = new StringBuilder(); - sb.append(alphabet.charAt(random.nextInt(6))); // start with letter - for (int i = 0; i < 7; i++) { - sb.append(alphabet.charAt(random.nextInt(alphabet.length()))); - } - return sb.toString(); + return GeneratedEndpoint.createPart(controller.random(true)); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index 65ca2028c5f..84746887d54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -11,13 +11,11 @@ import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory; import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement; -import java.security.SecureRandom; import java.time.Duration; import java.time.temporal.TemporalUnit; import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.Random; import java.util.concurrent.CopyOnWriteArrayList; import java.util.stream.Collectors; @@ -85,7 +83,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new BillingDatabaseMaintainer(controller, intervals.billingDatabaseMaintainer)); maintainers.add(new MeteringMonitorMaintainer(controller, intervals.meteringMonitorMaintainer, controller.serviceRegistry().resourceDatabase(), metric)); maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval)); - maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer, new SecureRandom())); + maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer)); } public Upgrader upgrader() { return upgrader; } @@ -197,15 +195,14 @@ public class ControllerMaintenance extends AbstractComponent { private static class SuccessFactorBaseline { - private final Double defaultSuccessFactorBaseline; private final Double deploymentMetricsMaintainerBaseline; private final Double trafficFractionUpdater; public SuccessFactorBaseline(SystemName system) { Objects.requireNonNull(system); - this.defaultSuccessFactorBaseline = 1.0; this.deploymentMetricsMaintainerBaseline = 0.90; this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65; } + } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index edcfcc317a7..a929a1d7af8 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -24,7 +24,6 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; import java.util.logging.Level; @@ -43,12 +42,10 @@ public class Upgrader extends ControllerMaintainer { private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; - private final Random random; public Upgrader(Controller controller, Duration interval) { super(controller, interval); this.curator = controller.curator(); - this.random = new Random(controller.clock().instant().toEpochMilli()); // Seed with clock for test determinism } /** @@ -78,7 +75,7 @@ public class Upgrader extends ControllerMaintainer { private InstanceList instances(DeploymentStatusList deploymentStatuses) { return InstanceList.from(deploymentStatuses) .withDeclaredJobs() - .shuffle(random) + .shuffle(controller().random(false)) .byIncreasingDeployedVersion() .unpinned(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java index bc977baf048..5770649c8b7 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java @@ -10,7 +10,9 @@ import com.yahoo.slime.Cursor; import com.yahoo.slime.Inspector; import com.yahoo.slime.Slime; import com.yahoo.slime.SlimeUtils; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; @@ -45,12 +47,15 @@ public class RoutingPolicySerializer { private static final String dnsZoneField = "dnsZone"; private static final String instanceEndpointsField = "rotations"; private static final String applicationEndpointsField = "applicationEndpoints"; - private static final String loadBalancerActiveField = "active"; private static final String globalRoutingField = "globalRouting"; private static final String agentField = "agent"; private static final String changedAtField = "changedAt"; private static final String statusField = "status"; private static final String privateOnlyField = "private"; + private static final String generatedEndpointsField = "generatedEndpoints"; + private static final String clusterPartField = "clusterPart"; + private static final String applicationPartField = "applicationPart"; + private static final String authMethodField = "authMethod"; public Slime toSlime(List<RoutingPolicy> routingPolicies) { var slime = new Slime(); @@ -69,6 +74,13 @@ public class RoutingPolicySerializer { policy.applicationEndpoints().forEach(endpointId -> applicationEndpointsArray.addString(endpointId.id())); globalRoutingToSlime(policy.routingStatus(), policyObject.setObject(globalRoutingField)); if ( ! policy.isPublic()) policyObject.setBool(privateOnlyField, true); + Cursor generatedEndpointsArray = policyObject.setArray(generatedEndpointsField); + policy.generatedEndpoints().forEach(generatedEndpoint -> { + Cursor generatedEndpointObject = generatedEndpointsArray.addObject(); + generatedEndpointObject.setString(clusterPartField, generatedEndpoint.clusterPart()); + generatedEndpointObject.setString(applicationPartField, generatedEndpoint.applicationPart()); + generatedEndpointObject.setString(authMethodField, authMethod(generatedEndpoint.authMethod())); + }); }); return slime; } @@ -86,6 +98,14 @@ public class RoutingPolicySerializer { ClusterSpec.Id.from(inspect.field(clusterField).asString()), ZoneId.from(inspect.field(zoneField).asString())); boolean isPublic = ! inspect.field(privateOnlyField).asBool(); + List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>(); + Inspector generatedEndpointsArray = inspect.field(generatedEndpointsField); + if (generatedEndpointsArray.valid()) { + generatedEndpointsArray.traverse((ArrayTraverser) (idx, generatedEndpointObject) -> + generatedEndpoints.add(new GeneratedEndpoint(generatedEndpointObject.field(clusterPartField).asString(), + generatedEndpointObject.field(applicationPartField).asString(), + authMethodFromSlime(generatedEndpointObject.field(authMethodField))))); + } policies.add(new RoutingPolicy(id, SlimeUtils.optionalString(inspect.field(canonicalNameField)).map(DomainName::of), SlimeUtils.optionalString(inspect.field(ipAddressField)), @@ -93,7 +113,8 @@ public class RoutingPolicySerializer { instanceEndpoints, applicationEndpoints, routingStatusFromSlime(inspect.field(globalRoutingField)), - isPublic)); + isPublic, + generatedEndpoints)); }); return Collections.unmodifiableList(policies); } @@ -111,4 +132,19 @@ public class RoutingPolicySerializer { return new RoutingStatus(status, agent, changedAt); } + private String authMethod(Endpoint.AuthMethod authMethod) { + return switch (authMethod) { + case token -> "token"; + case mtls -> "mtls"; + }; + } + + private Endpoint.AuthMethod authMethodFromSlime(Inspector field) { + return switch (field.asString()) { + case "token" -> Endpoint.AuthMethod.token; + case "mtls" -> Endpoint.AuthMethod.mtls; + default -> throw new IllegalArgumentException("Unknown auth method '" + field.asString() + "'"); + }; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java index 693275987c5..210b8df1447 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java @@ -1891,7 +1891,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler { object.setString("scope", endpointScopeString(endpoint.scope())); object.setString("routingMethod", routingMethodString(endpoint.routingMethod())); object.setBool("legacy", endpoint.legacy()); - object.setString("authMethod", endpoint.isTokenEndpoint() ? "token" : "mtls"); + switch (endpoint.authMethod()) { + case mtls -> object.setString("authMethod", "mtls"); + case token -> object.setString("authMethod", "token"); + } } private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java index a20d5497945..acd39b1c8b3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java @@ -22,13 +22,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData; import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName; -import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState; +import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget; import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedDirectTarget; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; import com.yahoo.vespa.hosted.controller.application.EndpointList; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; @@ -86,7 +87,7 @@ public class RoutingPolicies { } /** Read all routing policies for given application */ - private RoutingPolicyList read(TenantAndApplicationId application) { + public RoutingPolicyList read(TenantAndApplicationId application) { return db.readRoutingPolicies((instance) -> TenantAndApplicationId.from(instance).equals(application)) .values() .stream() @@ -112,7 +113,7 @@ public class RoutingPolicies { * Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if * routing configuration affecting given deployment has changed. */ - public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec) { + public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) { ApplicationId instance = deployment.applicationId(); List<LoadBalancer> loadBalancers = controller.serviceRegistry().configServer() .getLoadBalancers(instance, deployment.zoneId()); @@ -121,18 +122,17 @@ public class RoutingPolicies { Optional<TenantAndApplicationId> owner = ownerOf(allocation); try (var lock = db.lockRoutingPolicies()) { RoutingPolicyList applicationPolicies = read(TenantAndApplicationId.from(instance)); - RoutingPolicyList instancePolicies = applicationPolicies.instance(instance); RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(allocation.deployment); removeGlobalDnsUnreferencedBy(allocation, deploymentPolicies, lock); removeApplicationDnsUnreferencedBy(allocation, deploymentPolicies, lock); - instancePolicies = storePoliciesOf(allocation, instancePolicies, lock); + RoutingPolicyList instancePolicies = storePoliciesOf(allocation, applicationPolicies, generatedEndpoints, lock); instancePolicies = removePoliciesUnreferencedBy(allocation, instancePolicies, lock); - applicationPolicies = applicationPolicies.replace(instance, instancePolicies); + RoutingPolicyList updatedApplicationPolicies = applicationPolicies.replace(instance, instancePolicies); updateGlobalDnsOf(instancePolicies, Optional.of(deployment), inactiveZones, owner, lock); - updateApplicationDnsOf(applicationPolicies, inactiveZones, deployment, owner, lock); + updateApplicationDnsOf(updatedApplicationPolicies, inactiveZones, deployment, owner, lock); } } @@ -363,8 +363,8 @@ public class RoutingPolicies { * * @return the updated policies */ - private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) { - Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(instancePolicies.asMap()); + private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, List<GeneratedEndpoint> generatedEndpoints, @SuppressWarnings("unused") Mutex lock) { + Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(applicationPolicies.instance(allocation.deployment.applicationId()).asMap()); for (LoadBalancer loadBalancer : allocation.loadBalancers) { if (loadBalancer.hostname().isEmpty() && loadBalancer.ipAddress().isEmpty()) continue; var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId()); @@ -374,10 +374,17 @@ public class RoutingPolicies { allocation.instanceEndpointsOf(loadBalancer), allocation.applicationEndpointsOf(loadBalancer), RoutingStatus.DEFAULT, - loadBalancer.isPublic()); - // Preserve global routing status for existing policy + loadBalancer.isPublic(), + generatedEndpoints); + boolean addingGeneratedEndpoints = !generatedEndpoints.isEmpty() && (existingPolicy == null || existingPolicy.generatedEndpoints().isEmpty()); + if (addingGeneratedEndpoints) { + generatedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies)); + } if (existingPolicy != null) { - newPolicy = newPolicy.with(existingPolicy.routingStatus()); + newPolicy = newPolicy.with(existingPolicy.routingStatus()); // Always preserve routing status + if (!addingGeneratedEndpoints) { + newPolicy = newPolicy.with(existingPolicy.generatedEndpoints()); // Endpoints are generated once + } } updateZoneDnsOf(newPolicy, loadBalancer, allocation.deployment); policies.put(newPolicy.id(), newPolicy); @@ -402,7 +409,11 @@ public class RoutingPolicies { private void setPrivateDns(Endpoint endpoint, LoadBalancer loadBalancer, DeploymentId deploymentId) { if (loadBalancer.service().isEmpty()) return; - if (endpoint.isTokenEndpoint()) return; + boolean skipBasedOnAuthMethod = switch (endpoint.authMethod()) { + case token -> true; + case mtls -> false; + }; + if (skipBasedOnAuthMethod) return; controller.serviceRegistry().vpcEndpointService() .setPrivateDns(DomainName.of(endpoint.dnsName()), new ClusterId(deploymentId, endpoint.cluster()), @@ -723,4 +734,14 @@ public class RoutingPolicies { return ownerOf(allocation.deployment); } + private static void requireNonClashing(GeneratedEndpoint generatedEndpoint, RoutingPolicyList applicationPolicies) { + for (var policy : applicationPolicies) { + for (var ge : policy.generatedEndpoints()) { + if (ge.clusterPart().equals(generatedEndpoint.clusterPart())) { + throw new IllegalArgumentException(generatedEndpoint + " clashes with " + ge + " in " + policy.id()); + } + } + } + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java index fb8f5e8e129..d25a96b5ace 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java @@ -5,12 +5,13 @@ import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableSortedSet; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.zone.RoutingMethod; -import com.yahoo.text.Text; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint.Port; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; +import java.util.ArrayList; import java.util.List; import java.util.Objects; import java.util.Optional; @@ -29,11 +30,13 @@ public record RoutingPolicy(RoutingPolicyId id, Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, RoutingStatus routingStatus, - boolean isPublic) { + boolean isPublic, + List<GeneratedEndpoint> generatedEndpoints) { /** DO NOT USE. Public for serialization purposes */ public RoutingPolicy(RoutingPolicyId id, Optional<DomainName> canonicalName, Optional<String> ipAddress, Optional<String> dnsZone, - Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, RoutingStatus routingStatus, boolean isPublic) { + Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, RoutingStatus routingStatus, boolean isPublic, + List<GeneratedEndpoint> generatedEndpoints) { this.id = Objects.requireNonNull(id, "id must be non-null"); this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null"); this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null"); @@ -42,6 +45,7 @@ public record RoutingPolicy(RoutingPolicyId id, this.applicationEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(applicationEndpoints, "applicationEndpoints must be non-null")); this.routingStatus = Objects.requireNonNull(routingStatus, "status must be non-null"); this.isPublic = isPublic; + this.generatedEndpoints = List.copyOf(Objects.requireNonNull(generatedEndpoints, "generatedEndpoints must be non-null")); if (canonicalName.isEmpty() == ipAddress.isEmpty()) throw new IllegalArgumentException("Exactly 1 of canonicalName=%s and ipAddress=%s must be set".formatted( @@ -77,11 +81,16 @@ public record RoutingPolicy(RoutingPolicyId id, return instanceEndpoints; } - /** The application-level endpoints this participates in */ + /** The application-level endpoints this participates in */ public Set<EndpointId> applicationEndpoints() { return applicationEndpoints; } + /** The endpoints to generate for this policy, if any */ + public List<GeneratedEndpoint> generatedEndpoints() { + return generatedEndpoints; + } + /** Return status of routing */ public RoutingStatus routingStatus() { return routingStatus; @@ -100,19 +109,37 @@ public record RoutingPolicy(RoutingPolicyId id, /** Returns a copy of this with routing status set to given status */ public RoutingPolicy with(RoutingStatus routingStatus) { - return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic); + return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints); + } + + public RoutingPolicy with(List<GeneratedEndpoint> generatedEndpoints) { + return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints); } /** Returns the zone endpoints of this */ public List<Endpoint> zoneEndpointsIn(SystemName system, RoutingMethod routingMethod, boolean includeTokenEndpoint) { DeploymentId deployment = new DeploymentId(id.owner(), id.zone()); Endpoint zoneEndpoint = endpoint(routingMethod).target(id.cluster(), deployment).in(system); + List<Endpoint> endpoints = new ArrayList<>(); + endpoints.add(zoneEndpoint); if (includeTokenEndpoint) { - Endpoint tokenEndpoint = endpoint(routingMethod).target(id.cluster(), deployment).tokenEndpoint().in(system); - return List.of(zoneEndpoint, tokenEndpoint); - } else { - return List.of(zoneEndpoint); + Endpoint tokenEndpoint = endpoint(routingMethod).target(id.cluster(), deployment) + .authMethod(Endpoint.AuthMethod.token) + .in(system); + endpoints.add(tokenEndpoint); + } + for (var generatedEndpoint : generatedEndpoints) { + GeneratedEndpoint endpointToInclude = switch (generatedEndpoint.authMethod()) { + case token -> includeTokenEndpoint ? generatedEndpoint : null; + case mtls -> generatedEndpoint; + }; + if (endpointToInclude != null) { + endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment) + .generatedEndpoint(endpointToInclude) + .in(system)); + } } + return endpoints; } /** Returns the region endpoint of this */ @@ -133,17 +160,10 @@ public record RoutingPolicy(RoutingPolicyId id, return Objects.hash(id); } - @Override - public String toString() { - return Text.format("%s [instance endpoints: %s, application endpoints: %s%s], %s owned by %s, in %s", canonicalName, - instanceEndpoints, applicationEndpoints, - dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(), - id.zone().value()); - } - private Endpoint.EndpointBuilder endpoint(RoutingMethod routingMethod) { return Endpoint.of(id.owner()) .on(Port.fromRoutingMethod(routingMethod)) .routingMethod(routingMethod); } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index 2a7e4cb5c14..2e11a156dce 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer; import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint; import com.yahoo.vespa.hosted.controller.application.Endpoint; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; @@ -20,7 +21,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; import java.util.Set; -import java.util.stream.Collectors; /** * A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment. @@ -49,8 +49,8 @@ public abstract class DeploymentRoutingContext implements RoutingContext { } /** Configure routing for the deployment in this context, using given deployment spec */ - public final void configure(DeploymentSpec deploymentSpec) { - controller.policies().refresh(deployment, deploymentSpec); + public final void configure(DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) { + controller.policies().refresh(deployment, deploymentSpec, generatedEndpoints); } /** Routing method of this context */ diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index eabbdd76d5a..d9b95a53a0e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -66,6 +66,7 @@ import java.util.Arrays; import java.util.List; import java.util.Optional; import java.util.OptionalLong; +import java.util.Random; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; @@ -307,11 +308,11 @@ public final class ControllerTester { } public TenantName createTenant(String tenantName, Tenant.Type type) { - switch (type) { - case athenz: return createTenant(tenantName, "domain" + nextDomainId.getAndIncrement(), nextPropertyId.getAndIncrement()); - case cloud: return createCloudTenant(tenantName); - default: throw new UnsupportedOperationException(); - } + return switch (type) { + case athenz -> createTenant(tenantName, "domain" + nextDomainId.getAndIncrement(), nextPropertyId.getAndIncrement()); + case cloud -> createCloudTenant(tenantName); + default -> throw new UnsupportedOperationException(); + }; } public TenantName createTenant(String tenantName, String domainName, Long propertyId) { @@ -347,17 +348,13 @@ public final class ControllerTester { public Credentials credentialsFor(TenantName tenantName) { Tenant tenant = controller().tenants().require(tenantName); - switch (tenant.type()) { - case athenz: - return new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), - ((AthenzTenant) tenant).domain(), - OAuthCredentials.createForTesting("okta-access-token", "okta-identity-token")); - case cloud: - return new Credentials(new SimplePrincipal("dev")); - - default: - throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'"); - } + return switch (tenant.type()) { + case athenz -> new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")), + ((AthenzTenant) tenant).domain(), + OAuthCredentials.createForTesting("okta-access-token", "okta-identity-token")); + case cloud -> new Credentials(new SimplePrincipal("dev")); + default -> throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'"); + }; } public Application createApplication(ApplicationId id) { @@ -385,6 +382,7 @@ public final class ControllerTester { AthenzDbMock athensDb, ServiceRegistryMock serviceRegistry, FlagSource flagSource) { + Random random = new Random(serviceRegistry.clock().instant().toEpochMilli()); // Seed with clock for test determinism Controller controller = new Controller(curator, rotationsConfig, serviceRegistry.zoneRegistry().system().isPublic() ? @@ -395,7 +393,9 @@ public final class ControllerTester { serviceRegistry, new MetricsMock(), new SecretStoreMock(), new ControllerConfig.Builder().build(), - Sleeper.NOOP); + Sleeper.NOOP, + random, + random); // Calculate initial versions controller.updateVersionStatus(VersionStatus.compute(controller)); return controller; diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index dc96aa6c62c..23c029845bb 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -344,4 +344,36 @@ public class EndpointTest { tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamName(zone2))); } + @Test + public void generated_id() { + GeneratedEndpoint ge = new GeneratedEndpoint("cafed00d", "deadbeef", Endpoint.AuthMethod.mtls); + var deployment = new DeploymentId(instance1, ZoneId.from("prod", "us-north-1")); + var tests = Map.of( + // Zone endpoint in main, unlike named endpoints, this includes the scope symbol 'z' + "cafed00d.deadbeef.z.vespa.oath.cloud", + Endpoint.of(instance1).target(ClusterSpec.Id.from("c1"), deployment).generatedEndpoint(ge) + .routingMethod(RoutingMethod.sharedLayer4).on(Port.tls()).in(SystemName.main), + // Zone endpoint in public + "cafed00d.deadbeef.z.vespa-app.cloud", + Endpoint.of(instance1).target(ClusterSpec.Id.from("c1"), deployment).generatedEndpoint(ge) + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Global endpoint in public + "foo.deadbeef.g.vespa-app.cloud", + Endpoint.of(instance1).target(EndpointId.of("foo"), ClusterSpec.Id.from("c1"), List.of(deployment)) + .generatedEndpoint(ge) + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Global endpoint in public, with default ID + "deadbeef.g.vespa-app.cloud", + Endpoint.of(instance1).target(EndpointId.defaultId(), ClusterSpec.Id.from("c1"), List.of(deployment)) + .generatedEndpoint(ge) + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Application endpoint in public + "bar.deadbeef.a.vespa-app.cloud", + Endpoint.of(TenantAndApplicationId.from(instance1)).targetApplication(EndpointId.of("bar"), deployment) + .generatedEndpoint(ge) + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public) + ); + tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.dnsName())); + } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java index f94120241e7..a371677b82b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java @@ -10,7 +10,6 @@ import org.junit.jupiter.api.Test; import java.time.Duration; import java.util.List; -import java.util.Random; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -20,7 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; public class CertificatePoolMaintainerTest { private final ControllerTester tester = new ControllerTester(); - private final CertificatePoolMaintainer maintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1), new Random(4)); + private final CertificatePoolMaintainer maintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1)); @Test void new_certs_are_requested_until_limit() { @@ -41,17 +40,18 @@ public class CertificatePoolMaintainerTest { assertEquals( List.of( - new DnsNameStatus("*.c8868d4e.z.vespa.oath.cloud", "done"), - new DnsNameStatus("*.c8868d4e.g.vespa.oath.cloud", "done"), - new DnsNameStatus("*.c8868d4e.a.vespa.oath.cloud", "done") + new DnsNameStatus("*.f5549014.z.vespa.oath.cloud", "done"), + new DnsNameStatus("*.f5549014.g.vespa.oath.cloud", "done"), + new DnsNameStatus("*.f5549014.a.vespa.oath.cloud", "done") ), metadata.dnsNames()); - assertEquals("vespa.tls.preprovisioned.c8868d4e-cert", endpointCertificateProvider.certificateDetails(metadata.requestId()).cert_key_keyname()); - assertEquals("vespa.tls.preprovisioned.c8868d4e-key", endpointCertificateProvider.certificateDetails(metadata.requestId()).private_key_keyname()); + assertEquals("vespa.tls.preprovisioned.f5549014-cert", endpointCertificateProvider.certificateDetails(metadata.requestId()).cert_key_keyname()); + assertEquals("vespa.tls.preprovisioned.f5549014-key", endpointCertificateProvider.certificateDetails(metadata.requestId()).private_key_keyname()); } private void assertNumCerts(int n) { assertEquals(0.0, maintainer.maintain(), 0.0000001); assertEquals(n, tester.curator().readUnassignedCertificates().size()); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 24eb9f33d33..247ffe1de00 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -31,7 +31,6 @@ import java.time.temporal.ChronoUnit; import java.util.List; import java.util.Optional; import java.util.OptionalDouble; -import java.util.Random; import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1; @@ -49,7 +48,7 @@ public class EndpointCertificateMaintainerTest { private final ControllerTester tester = new ControllerTester(); private final SecretStoreMock secretStore = (SecretStoreMock) tester.controller().secretStore(); private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1)); - private final CertificatePoolMaintainer certificatePoolMaintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1), new Random(4)); + private final CertificatePoolMaintainer certificatePoolMaintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1)); private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()); @Test diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java index c1267ad5edf..f685c75bbe3 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java @@ -5,7 +5,9 @@ import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.zone.ZoneId; +import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.EndpointId; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy; import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId; import com.yahoo.vespa.hosted.controller.routing.RoutingStatus; @@ -44,7 +46,8 @@ public class RoutingPolicySerializerTest { Set.of(), Set.of(), RoutingStatus.DEFAULT, - false), + false, + List.of(new GeneratedEndpoint("deadbeef", "cafed00d", Endpoint.AuthMethod.mtls))), new RoutingPolicy(id2, Optional.of(HostName.of("long-and-ugly-name-2")), Optional.empty(), @@ -54,7 +57,8 @@ public class RoutingPolicySerializerTest { new RoutingStatus(RoutingStatus.Value.out, RoutingStatus.Agent.tenant, Instant.ofEpochSecond(123)), - true), + true, + List.of(new GeneratedEndpoint("cafed00d", "deadbeef", Endpoint.AuthMethod.token))), new RoutingPolicy(id1, Optional.empty(), Optional.of("127.0.0.1"), @@ -62,7 +66,8 @@ public class RoutingPolicySerializerTest { instanceEndpoints, applicationEndpoints, RoutingStatus.DEFAULT, - true)); + true, + List.of())); var serialized = serializer.fromSlime(owner, serializer.toSlime(policies)); assertEquals(policies.size(), serialized.size()); for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext(); ) { diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 0233db50ac6..783629c8f4a 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.routing; import ai.vespa.http.DomainName; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.application.api.ValidationId; import com.yahoo.config.provision.ApplicationId; @@ -20,6 +19,7 @@ import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.ControllerTester; import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record; import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type; @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList; import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; +import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; @@ -53,7 +54,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; -import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -262,11 +262,11 @@ public class RoutingPoliciesTest { context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones - Set<String> expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + List<String> expectedRecords = List.of( "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertEquals(4, tester.policiesOf(context1.instanceId()).size()); @@ -279,13 +279,13 @@ public class RoutingPoliciesTest { // Add 1 cluster in each zone and deploy tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + expectedRecords = List.of( "c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud" + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertEquals(6, tester.policiesOf(context1.instanceId()).size()); @@ -293,17 +293,17 @@ public class RoutingPoliciesTest { // Deploy another application tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2); context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", - "c2.app1.tenant1.us-west-1.vespa.oath.cloud", + expectedRecords = List.of( "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", - "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", "c0.app2.tenant1.us-west-1.vespa.oath.cloud", - "c1.app2.tenant1.us-west-1.vespa.oath.cloud" + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", + "c1.app2.tenant1.us-west-1.vespa.oath.cloud", + "c2.app1.tenant1.us-central-1.vespa.oath.cloud", + "c2.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords.stream().sorted().toList(), tester.recordNames().stream().sorted().toList()); assertEquals(4, tester.policiesOf(context2.instanceId()).size()); @@ -311,14 +311,14 @@ public class RoutingPoliciesTest { // Deploy removes cluster from app1 tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2); context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + expectedRecords = List.of( "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", "c0.app2.tenant1.us-central-1.vespa.oath.cloud", - "c1.app2.tenant1.us-central-1.vespa.oath.cloud", "c0.app2.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app2.tenant1.us-central-1.vespa.oath.cloud", "c1.app2.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); @@ -327,11 +327,11 @@ public class RoutingPoliciesTest { tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet() .forEach(zone -> tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone)); context2.flushDnsUpdates(); - expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "c1.app1.tenant1.us-west-1.vespa.oath.cloud", + expectedRecords = List.of( "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "c1.app1.tenant1.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "c1.app1.tenant1.us-central-1.vespa.oath.cloud", + "c1.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertTrue(tester.routingPolicies().read(context2.instanceId()).isEmpty(), "Removes stale routing policies " + context2.application()); @@ -350,11 +350,11 @@ public class RoutingPoliciesTest { context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); // Deployment creates records and policies for all clusters in all zones - Set<String> expectedRecords = Set.of( - "c0.app1.tenant1.us-west-1.vespa.oath.cloud", - "token-c0.app1.tenant1.us-west-1.vespa.oath.cloud", + List<String> expectedRecords = List.of( "c0.app1.tenant1.us-central-1.vespa.oath.cloud", - "token-c0.app1.tenant1.us-central-1.vespa.oath.cloud" + "c0.app1.tenant1.us-west-1.vespa.oath.cloud", + "token-c0.app1.tenant1.us-central-1.vespa.oath.cloud", + "token-c0.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); assertEquals(2, tester.policiesOf(context1.instanceId()).size()); @@ -367,7 +367,7 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(0, tester.controllerTester().controller().curator().readNameServiceQueue().requests().size()); - assertEquals(Set.of(), tester.recordNames()); + assertEquals(List.of(), tester.recordNames()); assertEquals(2, tester.policiesOf(context.instanceId()).size()); } @@ -409,15 +409,17 @@ public class RoutingPoliciesTest { .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - List<String> expectedRecords = List.of("c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "c0.app1.tenant1.gcp-us-south1-b.z.vespa-app.cloud", - "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", - "c0.app1.tenant1.gcp-us-south1.w.vespa-app.cloud", - "r0.app1.tenant1.g.vespa-app.cloud"); - assertEquals(Set.copyOf(expectedRecords), tester.recordNames()); + List<String> expectedRecords = List.of( + "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", + "c0.app1.tenant1.gcp-us-south1-b.z.vespa-app.cloud", + "c0.app1.tenant1.gcp-us-south1.w.vespa-app.cloud", + "r0.app1.tenant1.g.vespa-app.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); - assertEquals(List.of("lb-0--tenant1.app1.default--prod.aws-us-east-1c."), tester.recordDataOf(Record.Type.CNAME, expectedRecords.get(0))); - assertEquals(List.of("10.0.0.0"), tester.recordDataOf(Record.Type.A, expectedRecords.get(1))); + assertEquals(List.of("lb-0--tenant1.app1.default--prod.aws-us-east-1c."), tester.recordDataOf(Record.Type.CNAME, expectedRecords.get(1))); + assertEquals(List.of("10.0.0.0"), tester.recordDataOf(Record.Type.A, expectedRecords.get(2))); assertEquals(List.of("weighted/10.0.0.0/prod.gcp-us-south1-b/1"), tester.recordDataOf(Record.Type.DIRECT, expectedRecords.get(3))); assertEquals(List.of("latency/c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud/dns-zone-1/prod.aws-us-east-1c", "latency/c0.app1.tenant1.gcp-us-south1.w.vespa-app.cloud/ignored/prod.gcp-us-south1-b"), @@ -443,11 +445,12 @@ public class RoutingPoliciesTest { tester.assertTargets(context.instanceId(), EndpointId.defaultId(), ClusterSpec.Id.from("default"), 0, Map.of(zone1, 1L, zone2, 1L)); - assertEquals(Set.of("app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", - "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", - "app1.tenant1.aws-us-east-1.w.vespa-app.cloud", - "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "app1.tenant1.g.vespa-app.cloud"), + assertEquals(List.of("app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", + "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "app1.tenant1.aws-us-east-1.w.vespa-app.cloud", + "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", + "app1.tenant1.g.vespa-app.cloud" + ), tester.recordNames(), "Registers expected DNS names"); } @@ -471,7 +474,7 @@ public class RoutingPoliciesTest { // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(context.instanceId()).size()); - assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); + assertEquals(List.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames()); } @Test @@ -482,7 +485,7 @@ public class RoutingPoliciesTest { context.submit(applicationPackage).deploy(); var zone = ZoneId.from("dev", "us-east-1"); tester.controllerTester().setRoutingMethod(List.of(zone), RoutingMethod.exclusive); - var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); + var prodRecords = List.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud"); assertEquals(prodRecords, tester.recordNames()); // Deploy to dev under different instance @@ -494,7 +497,8 @@ public class RoutingPoliciesTest { // Routing policy is created and DNS is updated assertEquals(1, tester.policiesOf(devContext.instanceId()).size()); - assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames()); + assertEquals(Stream.concat(prodRecords.stream(), Stream.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")).sorted().toList(), + tester.recordNames()); } @Test @@ -510,7 +514,7 @@ public class RoutingPoliciesTest { // Application is deployed context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); - var expectedRecords = Set.of( + var expectedRecords = List.of( "c0.app1.tenant1.us-west-1.vespa.oath.cloud" ); assertEquals(expectedRecords, tester.recordNames()); @@ -557,8 +561,8 @@ public class RoutingPoliciesTest { app.deploy(); // TXT records are cleaned up as we go—the last challenge is the last to go here, and we must flush it ourselves. - assertEquals(Set.of("a.t.aws-us-east-1a.vespa.oath.cloud", - "challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), + assertEquals(List.of("a.t.aws-us-east-1a.vespa.oath.cloud", + "challenge--a.t.aws-us-east-1a.vespa.oath.cloud"), tester.recordNames()); app.flushDnsUpdates(); assertEquals(Set.of(new Record(Type.CNAME, @@ -773,7 +777,7 @@ public class RoutingPoliciesTest { tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out, RoutingStatus.Agent.tenant); } catch (IllegalArgumentException e) { - assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive]", e.getMessage()); + assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive, authMethod=mtls]", e.getMessage()); } context.flushDnsUpdates(); tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2); @@ -841,9 +845,9 @@ public class RoutingPoliciesTest { // Application endpoints are not created until production jobs run betaContext.submit(applicationPackage) .runJob(DeploymentContext.systemTest); - assertEquals(Set.of("beta.app1.tenant1.us-east-1.test.vespa.oath.cloud"), tester.recordNames()); + assertEquals(List.of("beta.app1.tenant1.us-east-1.test.vespa.oath.cloud"), tester.recordNames()); betaContext.runJob(DeploymentContext.stagingTest); - assertEquals(Set.of("beta.app1.tenant1.us-east-3.staging.vespa.oath.cloud"), tester.recordNames()); + assertEquals(List.of("beta.app1.tenant1.us-east-3.staging.vespa.oath.cloud"), tester.recordNames()); // Deploy both instances betaContext.completeRollout(); @@ -958,7 +962,7 @@ public class RoutingPoliciesTest { tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant); fail("Expected exception"); } catch (IllegalArgumentException e) { - assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive]", + assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive, authMethod=mtls]", e.getMessage()); } @@ -1004,6 +1008,53 @@ public class RoutingPoliciesTest { "Policies removed"); } + @Test + public void generated_zone_endpoints() { + var tester = new RoutingPoliciesTester(SystemName.Public); + var context = tester.newDeploymentContext("tenant1", "app1", "default"); + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); + + // Deploy application + int clustersPerZone = 1; + var zone1 = ZoneId.from("prod", "aws-us-east-1c"); + var zone2 = ZoneId.from("prod", "aws-eu-west-1a"); + ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region()) + .region(zone2.region()) + .build(); + tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2); + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + + // Deployment creates generated zone names + List<String> expectedRecords = List.of( + "a9c8c045.cafed00d.z.vespa-app.cloud", + "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", + "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", + "e144a11b.cafed00d.z.vespa-app.cloud" + ); + assertEquals(expectedRecords, tester.recordNames()); + assertEquals(2, tester.policiesOf(context.instanceId()).size()); + for (var zone : List.of(zone1, zone2)) { + EndpointList endpoints = tester.controllerTester().controller().routing().readEndpointsOf(context.deploymentIdIn(zone)); + assertEquals(1, endpoints.generated().size()); + } + + // Next deployment does not change generated names + context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); + assertEquals(expectedRecords, tester.recordNames()); + } + + private void addCertificateToPool(String id, UnassignedCertificate.State state, RoutingPoliciesTester tester) { + EndpointCertificateMetadata cert = new EndpointCertificateMetadata("testKey", "testCert", 1, 0, + "request-id", + Optional.of("leaf-request-uuid"), + List.of("name1", "name2"), + "", Optional.empty(), + Optional.empty(), Optional.of(id)); + UnassignedCertificate pooledCert = new UnassignedCertificate(cert, state); + tester.controllerTester().controller().curator().writeUnassignedCertificate(pooledCert); + } + /** Returns an application package builder that satisfies requirements for a directly routed endpoint */ private static ApplicationPackageBuilder applicationPackageBuilder() { return new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), @@ -1113,11 +1164,13 @@ public class RoutingPoliciesTest { return tester.controller().routing().policies().read(instance); } - private Set<String> recordNames() { + private List<String> recordNames() { return tester.controllerTester().nameService().records().stream() .map(Record::name) .map(RecordName::asString) - .collect(Collectors.toSet()); + .distinct() + .sorted() + .toList(); } private Set<String> aliasDataOf(String name) { |