summaryrefslogtreecommitdiffstats
path: root/controller-server
diff options
context:
space:
mode:
authorMartin Polden <mpolden@mpolden.no>2023-07-04 11:57:49 +0200
committerMartin Polden <mpolden@mpolden.no>2023-07-05 12:52:00 +0200
commitd56134d93b7d62f5e96b688185d3d5b64d0bf942 (patch)
treecfb60fe3ef1a99288be9974d0eb65984a6f5d777 /controller-server
parent8fc733d22134a4645e6b3bb4cdde524cb251f5ae (diff)
Support anonymized endpoints
Diffstat (limited to 'controller-server')
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java22
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java20
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java19
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java123
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java3
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java10
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java34
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java17
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java7
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java5
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java40
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java5
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java47
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java54
-rw-r--r--controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java6
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java34
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java32
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java14
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java3
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java11
-rw-r--r--controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java169
21 files changed, 475 insertions, 200 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index adadcab3270..fdb27ba49a3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -55,6 +55,7 @@ import com.yahoo.vespa.hosted.controller.application.Deployment;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics;
import com.yahoo.vespa.hosted.controller.application.DeploymentMetrics.Warning;
import com.yahoo.vespa.hosted.controller.application.DeploymentQuotaCalculator;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.application.QuotaUsage;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
@@ -88,6 +89,7 @@ import java.security.cert.X509Certificate;
import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -150,6 +152,7 @@ public class ApplicationController {
private final ListFlag<String> incompatibleVersions;
private final BillingController billingController;
private final ListFlag<String> cloudAccountsFlag;
+
private final Map<DeploymentId, com.yahoo.vespa.hosted.controller.api.integration.configserver.Application> deploymentInfo = new ConcurrentHashMap<>();
ApplicationController(Controller controller, CuratorDb curator, AccessControl accessControl, Clock clock,
@@ -651,7 +654,8 @@ public class ApplicationController {
DeploymentId deployment = new DeploymentId(application, zone);
// Routing and metadata may have changed, so we need to refresh state after deployment, even if deployment fails.
interface CleanCloseable extends AutoCloseable { void close(); }
- try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage)) {
+ List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>();
+ try (CleanCloseable postDeployment = () -> updateRoutingAndMeta(deployment, applicationPackage, generatedEndpoints)) {
Optional<DockerImage> dockerImageRepo = Optional.ofNullable(
dockerImageRepoFlag
.with(FetchVector.Dimension.ZONE_ID, zone.value())
@@ -680,8 +684,16 @@ public class ApplicationController {
}
Supplier<Optional<CloudAccount>> cloudAccount = () -> decideCloudAccountOf(deployment, applicationPackage.truncatedPackage().deploymentSpec());
List<DataplaneTokenVersions> dataplaneTokenVersions = controller.dataplaneTokenService().listTokens(application.tenant());
+ Supplier<Optional<EndpointCertificateMetadata>> endpointCertificateMetadataWrapper = () -> {
+ Optional<EndpointCertificateMetadata> data = endpointCertificateMetadata.get();
+ // TODO(mpolden): Pass these endpoints to config server as part of the deploy call. This will let the
+ // application know which endpoints are mTLS and which are token-based
+ data.flatMap(EndpointCertificateMetadata::randomizedId)
+ .ifPresent(applicationPart -> generatedEndpoints.addAll(controller.routing().generateEndpoints(applicationPart, deployment.applicationId())));
+ return data;
+ };
DeploymentData deploymentData = new DeploymentData(application, zone, applicationPackage::zipStream, platform,
- endpoints, endpointCertificateMetadata, dockerImageRepo, domain,
+ endpoints, endpointCertificateMetadataWrapper, dockerImageRepo, domain,
deploymentQuota, tenantSecretStores, operatorCertificates, cloudAccount, dataplaneTokenVersions, dryRun);
ConfigServer.PreparedApplication preparedApplication = configServer.deploy(deploymentData);
@@ -689,9 +701,9 @@ public class ApplicationController {
}
}
- private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data) {
+ private void updateRoutingAndMeta(DeploymentId id, ApplicationPackageStream data, List<GeneratedEndpoint> generatedEndpoints) {
if (id.applicationId().instance().isTester()) return;
- controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec());
+ controller.routing().of(id).configure(data.truncatedPackage().deploymentSpec(), generatedEndpoints);
if ( ! id.zoneId().environment().isManuallyDeployed()) return;
controller.applications().applicationStore().putMeta(id, clock.instant(), data.truncatedPackage().metaDataZip());
}
@@ -906,7 +918,7 @@ public class ApplicationController {
DeploymentId id = new DeploymentId(instanceId, zone);
interface CleanCloseable extends AutoCloseable { void close(); }
try (CleanCloseable postDeactivation = () -> {
- application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec()));
+ application.ifPresent(app -> controller.routing().of(id).configure(app.get().deploymentSpec(), List.of()));
if (id.zoneId().environment().isManuallyDeployed())
applicationStore.putMetaTombstone(id, clock.instant());
if ( ! id.zoneId().environment().isTest())
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 287cfaa41b8..f6bcbc9828b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -27,8 +27,8 @@ import com.yahoo.vespa.hosted.controller.notification.NotificationsDb;
import com.yahoo.vespa.hosted.controller.notification.Notifier;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import com.yahoo.vespa.hosted.controller.persistence.JobControlFlags;
-import com.yahoo.vespa.hosted.controller.security.AccessControl;
import com.yahoo.vespa.hosted.controller.restapi.dataplanetoken.DataplaneTokenService;
+import com.yahoo.vespa.hosted.controller.security.AccessControl;
import com.yahoo.vespa.hosted.controller.support.access.SupportAccessControl;
import com.yahoo.vespa.hosted.controller.versions.OsVersion;
import com.yahoo.vespa.hosted.controller.versions.OsVersionStatus;
@@ -38,12 +38,14 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
import com.yahoo.yolean.concurrent.Sleeper;
+import java.security.SecureRandom;
import java.time.Clock;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
+import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
import java.util.function.Function;
@@ -91,6 +93,8 @@ public class Controller extends AbstractComponent {
private final Notifier notifier;
private final MailVerifier mailVerifier;
private final DataplaneTokenService dataplaneTokenService;
+ private final Random random;
+ private final Random secureRandom; // Type is Random to allow for test determinism
/**
* Creates a controller
@@ -102,13 +106,14 @@ public class Controller extends AbstractComponent {
MavenRepository mavenRepository, ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore,
ControllerConfig controllerConfig) {
this(curator, rotationsConfig, accessControl, flagSource,
- mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT);
+ mavenRepository, serviceRegistry, metric, secretStore, controllerConfig, Sleeper.DEFAULT, new Random(),
+ new SecureRandom());
}
public Controller(CuratorDb curator, RotationsConfig rotationsConfig, AccessControl accessControl,
FlagSource flagSource, MavenRepository mavenRepository,
ServiceRegistry serviceRegistry, Metric metric, SecretStore secretStore,
- ControllerConfig controllerConfig, Sleeper sleeper) {
+ ControllerConfig controllerConfig, Sleeper sleeper, Random random, Random secureRandom) {
this.curator = Objects.requireNonNull(curator, "Curator cannot be null");
this.serviceRegistry = Objects.requireNonNull(serviceRegistry, "ServiceRegistry cannot be null");
this.zoneRegistry = Objects.requireNonNull(serviceRegistry.zoneRegistry(), "ZoneRegistry cannot be null");
@@ -119,6 +124,8 @@ public class Controller extends AbstractComponent {
this.metric = Objects.requireNonNull(metric, "Metric cannot be null");
this.controllerConfig = Objects.requireNonNull(controllerConfig, "ControllerConfig cannot be null");
this.secretStore = Objects.requireNonNull(secretStore, "SecretStore cannot be null");
+ this.random = Objects.requireNonNull(random, "Random cannot be null");
+ this.secureRandom = Objects.requireNonNull(secureRandom, "SecureRandom cannot be null");
nameServiceForwarder = new NameServiceForwarder(curator);
jobController = new JobController(this);
@@ -362,4 +369,11 @@ public class Controller extends AbstractComponent {
public DataplaneTokenService dataplaneTokenService() {
return dataplaneTokenService;
}
+
+ /** Returns a random number generator. If secure is true, this returns a {@link SecureRandom} suitable for
+ * cryptographic purposes */
+ public Random random(boolean secure) {
+ return secure ? secureRandom : random;
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index d0b34b6094d..ceac681255b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
import com.yahoo.vespa.hosted.controller.application.Endpoint.Scope;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
@@ -44,6 +45,7 @@ import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
@@ -75,6 +77,7 @@ public class RoutingController {
private final RoutingPolicies routingPolicies;
private final RotationRepository rotationRepository;
private final BooleanFlag createTokenEndpoint;
+ private final BooleanFlag randomizedEndpoints;
public RoutingController(Controller controller, RotationsConfig rotationsConfig) {
this.controller = Objects.requireNonNull(controller, "controller must be non-null");
@@ -83,6 +86,7 @@ public class RoutingController {
controller.applications(),
controller.curator());
this.createTokenEndpoint = Flags.ENABLE_DATAPLANE_PROXY.bindTo(controller.flagSource());
+ this.randomizedEndpoints = Flags.RANDOMIZED_ENDPOINT_NAMES.bindTo(controller.flagSource());
}
/** Create a routing context for given deployment */
@@ -138,6 +142,7 @@ public class RoutingController {
/** Returns endpoints declared in {@link DeploymentSpec} for given application */
public EndpointList declaredEndpointsOf(Application application) {
+ // TODO(mpolden): Add generated endpoints for global and application scopes. Requires reading routing polices here
Set<Endpoint> endpoints = new LinkedHashSet<>();
DeploymentSpec deploymentSpec = application.deploymentSpec();
for (var spec : deploymentSpec.instances()) {
@@ -169,7 +174,6 @@ public class RoutingController {
t -> t.weight()));
ZoneId zone = deployments.keySet().iterator().next().zoneId(); // Where multiple zones are possible, they all have the same routing method.
- // Application endpoints are only supported when using direct routing methods
RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive;
endpoints.add(Endpoint.of(application.id())
.targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
@@ -358,6 +362,19 @@ public class RoutingController {
Optional.of(application.id())));
}
+ /** Generate endpoints for all authenticaiton methods, using given application part */
+ public List<GeneratedEndpoint> generateEndpoints(String applicationPart, ApplicationId instance) {
+ boolean enabled = randomizedEndpoints.with(FetchVector.Dimension.APPLICATION_ID, instance.serializedForm()).value();
+ if (!enabled) {
+ return List.of();
+ }
+ return Arrays.stream(Endpoint.AuthMethod.values())
+ .map(method -> new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)),
+ applicationPart,
+ method))
+ .toList();
+ }
+
/**
* Assigns one or more global rotations to given application, if eligible. The given application is implicitly
* stored, ensuring that the assigned rotation(s) are persisted when this returns.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index f55bb0dab6f..a3381819778 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -44,11 +44,12 @@ public class Endpoint {
private final Scope scope;
private final boolean legacy;
private final RoutingMethod routingMethod;
- private boolean tokenEndpoint;
+ private final AuthMethod authMethod;
+ private final boolean generated;
private Endpoint(TenantAndApplicationId application, Optional<InstanceName> instanceName, EndpointId id,
ClusterSpec.Id cluster, URI url, List<Target> targets, Scope scope, Port port, boolean legacy,
- RoutingMethod routingMethod, boolean certificateName, boolean tokenEndpoint) {
+ RoutingMethod routingMethod, boolean certificateName, AuthMethod authMethod, boolean generated) {
Objects.requireNonNull(application, "application must be non-null");
Objects.requireNonNull(instanceName, "instanceName must be non-null");
Objects.requireNonNull(cluster, "cluster must be non-null");
@@ -57,6 +58,7 @@ public class Endpoint {
Objects.requireNonNull(scope, "scope must be non-null");
Objects.requireNonNull(port, "port must be non-null");
Objects.requireNonNull(routingMethod, "routingMethod must be non-null");
+ Objects.requireNonNull(authMethod, "authMethod must be non-null");
this.id = requireEndpointId(id, scope, certificateName);
this.cluster = requireCluster(cluster, certificateName);
this.instance = requireInstance(instanceName, scope);
@@ -65,7 +67,8 @@ public class Endpoint {
this.scope = requireScope(scope, routingMethod);
this.legacy = legacy;
this.routingMethod = routingMethod;
- this.tokenEndpoint = tokenEndpoint;
+ this.authMethod = authMethod;
+ this.generated = generated;
}
/**
@@ -135,6 +138,11 @@ public class Endpoint {
return routingMethod.isShared() && scope == Scope.global;
}
+ /** Returns whether this endpoint is generated by the system */
+ public boolean generated() {
+ return generated;
+ }
+
/** Returns the upstream name of given deployment. This *must* match what the routing layer generates */
public String upstreamName(DeploymentId deployment) {
if (!routingMethod.isShared()) throw new IllegalArgumentException("Routing method " + routingMethod + " does not have upstream name");
@@ -156,7 +164,7 @@ public class Endpoint {
@Override
public String toString() {
- return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s]", url, scope, legacy, routingMethod);
+ return Text.format("endpoint %s [scope=%s, legacy=%s, routingMethod=%s, authMethod=%s]", url, scope, legacy, routingMethod, authMethod);
}
private static String endpointOrClusterAsString(EndpointId id, ClusterSpec.Id cluster) {
@@ -164,24 +172,39 @@ public class Endpoint {
}
private static URI createUrl(String name, TenantAndApplicationId application, Optional<InstanceName> instance,
- List<Target> targets, Scope scope, SystemName system, Port port, boolean legacyRegionalUrl) {
+ List<Target> targets, Scope scope, SystemName system, Port port,
+ Optional<GeneratedEndpoint> generated) {
String separator = ".";
String portPart = port.isDefault() ? "" : ":" + port.port;
+ final String subdomain;
+ if (generated.isPresent()) {
+ subdomain = generatedPart(generated.get(), name, scope, separator);
+ } else {
+ subdomain = sanitize(namePart(name, separator)) +
+ systemPart(system, separator) +
+ sanitize(instancePart(instance, separator)) +
+ sanitize(application.application().value()) +
+ separator +
+ sanitize(application.tenant().value());
+ }
return URI.create("https://" +
- sanitize(namePart(name, separator)) +
- systemPart(system, separator) +
- sanitize(instancePart(instance, separator)) +
- sanitize(application.application().value()) +
- separator +
- sanitize(application.tenant().value()) +
+ subdomain +
"." +
- scopePart(scope, targets, system, legacyRegionalUrl) +
+ scopePart(scope, targets, system, generated) +
dnsSuffix(system) +
portPart +
"/");
}
+ private static String generatedPart(GeneratedEndpoint generated, String name, Scope scope, String separator) {
+ if (scope.multiDeployment()) {
+ // Endpoints with these scopes have a name part that is explicitly configured through deployment.xml
+ return sanitize(namePart(name, separator)) + generated.applicationPart();
+ }
+ return generated.clusterPart() + separator + generated.applicationPart();
+ }
+
private static String sanitize(String part) { // TODO: Reject reserved words
return part.replace('_', '-');
}
@@ -191,24 +214,23 @@ public class Endpoint {
return name + separator;
}
- private static String scopePart(Scope scope, List<Target> targets, SystemName system, boolean legacyRegion) {
- String scopeSymbol = scopeSymbol(scope, system, legacyRegion);
+ private static String scopePart(Scope scope, List<Target> targets, SystemName system, Optional<GeneratedEndpoint> generated) {
+ String scopeSymbol = scopeSymbol(scope, system, generated);
if (scope == Scope.global) return scopeSymbol;
- if (scope == Scope.application && ! legacyRegion) return scopeSymbol;
+ if (scope == Scope.application) return scopeSymbol;
+ if (generated.isPresent()) return scopeSymbol;
ZoneId zone = targets.stream().map(target -> target.deployment.zoneId()).min(comparing(ZoneId::value)).get();
String region = zone.region().value();
- boolean skipEnvironment = zone.environment().isProduction();
- String environment = skipEnvironment ? "" : "." + zone.environment().value();
+ String environment = zone.environment().isProduction() ? "" : "." + zone.environment().value();
if (system.isPublic()) {
return region + environment + "." + scopeSymbol;
}
return region + (scopeSymbol.isEmpty() ? "" : "-" + scopeSymbol) + environment;
}
- private static String scopeSymbol(Scope scope, SystemName system, boolean legacyRegion) {
- if (legacyRegion) return "r";
- if (system.isPublic()) {
+ private static String scopeSymbol(Scope scope, SystemName system, Optional<GeneratedEndpoint> generated) {
+ if (system.isPublic() || generated.isPresent()) {
return switch (scope) {
case zone -> "z";
case weighted -> "w";
@@ -347,8 +369,9 @@ public class Endpoint {
return targets;
}
- public boolean isTokenEndpoint() {
- return tokenEndpoint;
+ /** Returns the authentication method of this endpoint */
+ public AuthMethod authMethod() {
+ return authMethod;
}
/** An endpoint's scope */
@@ -381,22 +404,25 @@ public class Endpoint {
}
+ /** An endpoint's authentication method */
+ public enum AuthMethod {
+ mtls,
+ token,
+ }
+
/** Represents an endpoint's HTTP port */
- public static class Port {
+ public record Port(int port) {
private static final Port TLS_DEFAULT = new Port(443);
- private final int port;
-
- private Port(int port) {
+ public Port {
if (port < 1 || port > 65535) {
throw new IllegalArgumentException("Port must be between 1 and 65535, got " + port);
}
- this.port = port;
}
private boolean isDefault() {
- return port == 443;
+ return port == TLS_DEFAULT.port;
}
/** Returns the default HTTPS port */
@@ -407,12 +433,7 @@ public class Endpoint {
/** Returns default port for the given routing method */
public static Port fromRoutingMethod(RoutingMethod method) {
if (method.isDirect()) return Port.tls();
- return Port.tls(4443);
- }
-
- /** Create a HTTPS port */
- public static Port tls(int port) {
- return new Port(port);
+ return new Port(4443);
}
}
@@ -475,14 +496,15 @@ public class Endpoint {
private RoutingMethod routingMethod = RoutingMethod.sharedLayer4;
private boolean legacy = false;
private boolean certificateName = false;
- private boolean tokenEndpoint = false;
+ private AuthMethod authMethod = AuthMethod.mtls;
+ private Optional<GeneratedEndpoint> generated = Optional.empty();
private EndpointBuilder(TenantAndApplicationId application, Optional<InstanceName> instance) {
this.application = Objects.requireNonNull(application);
this.instance = Objects.requireNonNull(instance);
}
- /** Sets the deployment target for this */
+ /** Sets the zone target for this */
public EndpointBuilder target(ClusterSpec.Id cluster, DeploymentId deployment) {
this.cluster = cluster;
this.scope = requireUnset(Scope.zone);
@@ -543,8 +565,9 @@ public class Endpoint {
return this;
}
- public EndpointBuilder tokenEndpoint() {
- this.tokenEndpoint = true;
+ /** Sets the valid authentication method supported by this */
+ public EndpointBuilder authMethod(AuthMethod authMethod) {
+ this.authMethod = authMethod;
return this;
}
@@ -572,23 +595,32 @@ public class Endpoint {
return this;
}
+ /** Sets the generated ID to use when building this */
+ public EndpointBuilder generatedEndpoint(GeneratedEndpoint generated) {
+ this.generated = Optional.of(generated);
+ this.authMethod = generated.authMethod();
+ return this;
+ }
+
/** Sets the system that owns this */
public Endpoint in(SystemName system) {
if (system.isPublic() && routingMethod != RoutingMethod.exclusive) {
throw new IllegalArgumentException("Public system only supports routing method " + RoutingMethod.exclusive);
}
- if (routingMethod.isDirect() && !port.isDefault()) {
- throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port");
- }
- String prefix = tokenEndpoint ? "token-" : "";
- URI url = createUrl(prefix + endpointOrClusterAsString(endpointId, cluster),
+ String prefix = authMethod == AuthMethod.token ? "token-" : "";
+ String name = endpointOrClusterAsString(endpointId, Objects.requireNonNull(cluster, "cluster must be non-null"));
+ URI url = createUrl(prefix + name,
Objects.requireNonNull(application, "application must be non-null"),
Objects.requireNonNull(instance, "instance must be non-null"),
Objects.requireNonNull(targets, "targets must be non-null"),
Objects.requireNonNull(scope, "scope must be non-null"),
Objects.requireNonNull(system, "system must be non-null"),
Objects.requireNonNull(port, "port must be non-null"),
- false);
+ Objects.requireNonNull(generated)
+ );
+ if (routingMethod.isDirect() && !port.isDefault()) {
+ throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port");
+ }
return new Endpoint(application,
instance,
endpointId,
@@ -600,7 +632,8 @@ public class Endpoint {
legacy,
routingMethod,
certificateName,
- tokenEndpoint);
+ authMethod,
+ generated.isPresent());
}
private Scope requireUnset(Scope scope) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java
index de1429d88af..b7ca8587efa 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointId.java
@@ -4,8 +4,7 @@ package com.yahoo.vespa.hosted.controller.application;
import java.util.Objects;
/**
- * A type to represent the ID of an endpoint. This is typically the first part of
- * an endpoint name.
+ * A user-specified endpoint ID. This is typically the first part of an endpoint name.
*
* @author ogronnesby
*/
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
index 3da9065b52d..5026fea7847 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
@@ -64,6 +64,11 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
return matching(Endpoint::legacy);
}
+ /** Returns the subset of endpoints generated by the system */
+ public EndpointList generated() {
+ return matching(Endpoint::generated);
+ }
+
/** Returns the subset of endpoints that require a rotation */
public EndpointList requiresRotation() {
return matching(Endpoint::requiresRotation);
@@ -88,4 +93,9 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
return new EndpointList(endpoints, false);
}
+ @Override
+ public String toString() {
+ return asList().toString();
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java
new file mode 100644
index 00000000000..dd6f4e5111d
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java
@@ -0,0 +1,34 @@
+package com.yahoo.vespa.hosted.controller.application;
+
+import ai.vespa.validation.Validation;
+
+import java.util.random.RandomGenerator;
+import java.util.regex.Pattern;
+
+/**
+ * A system-generated endpoint, where the cluster and application parts are randomly generated. These become the
+ * first and second part of an endpoint name. See {@link Endpoint}.
+ *
+ * @author mpolden
+ */
+public record GeneratedEndpoint(String clusterPart, String applicationPart, Endpoint.AuthMethod authMethod) {
+
+ private static final Pattern PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$");
+
+ public GeneratedEndpoint {
+ Validation.requireMatch(clusterPart, "Cluster part", PART_PATTERN);
+ Validation.requireMatch(applicationPart, "Application part", PART_PATTERN);
+ }
+
+ /** Create a new endpoint part, using random as a source of randomness */
+ public static String createPart(RandomGenerator random) {
+ String alphabet = "abcdef0123456789";
+ StringBuilder sb = new StringBuilder();
+ sb.append(alphabet.charAt(random.nextInt(6))); // Start with letter
+ for (int i = 0; i < 7; i++) {
+ sb.append(alphabet.charAt(random.nextInt(alphabet.length())));
+ }
+ return sb.toString();
+ }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java
index 82d4f068d6d..46d19b627cc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java
@@ -14,9 +14,10 @@ import com.yahoo.vespa.flags.StringFlag;
import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
-import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate;
+import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate;
import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
import java.time.Duration;
@@ -27,7 +28,6 @@ import java.util.OptionalInt;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
-import java.util.random.RandomGenerator;
import java.util.stream.Collectors;
/**
@@ -39,7 +39,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer {
private static final Logger log = Logger.getLogger(CertificatePoolMaintainer.class.getName());
- private final RandomGenerator random;
private final CuratorDb curator;
private final SecretStore secretStore;
private final EndpointCertificateProvider endpointCertificateProvider;
@@ -50,7 +49,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer {
private final StringFlag endpointCertificateAlgo;
private final BooleanFlag useAlternateCertProvider;
- public CertificatePoolMaintainer(Controller controller, Metric metric, Duration interval, RandomGenerator random) {
+ public CertificatePoolMaintainer(Controller controller, Metric metric, Duration interval) {
super(controller, interval, null, Set.of(SystemName.Public, SystemName.PublicCd));
this.controller = controller;
this.secretStore = controller.secretStore();
@@ -61,7 +60,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer {
this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider();
this.metric = metric;
this.dnsSuffix = Endpoint.dnsSuffix(controller.system());
- this.random = random;
}
protected double maintain() {
@@ -129,12 +127,7 @@ public class CertificatePoolMaintainer extends ControllerMaintainer {
}
private String generateRandomId() {
- String alphabet = "abcdef0123456789";
- StringBuilder sb = new StringBuilder();
- sb.append(alphabet.charAt(random.nextInt(6))); // start with letter
- for (int i = 0; i < 7; i++) {
- sb.append(alphabet.charAt(random.nextInt(alphabet.length())));
- }
- return sb.toString();
+ return GeneratedEndpoint.createPart(controller.random(true));
}
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 65ca2028c5f..84746887d54 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -11,13 +11,11 @@ import com.yahoo.vespa.hosted.controller.Controller;
import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactory;
import com.yahoo.vespa.hosted.controller.api.integration.user.UserManagement;
-import java.security.SecureRandom;
import java.time.Duration;
import java.time.temporal.TemporalUnit;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
-import java.util.Random;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
@@ -85,7 +83,7 @@ public class ControllerMaintenance extends AbstractComponent {
maintainers.add(new BillingDatabaseMaintainer(controller, intervals.billingDatabaseMaintainer));
maintainers.add(new MeteringMonitorMaintainer(controller, intervals.meteringMonitorMaintainer, controller.serviceRegistry().resourceDatabase(), metric));
maintainers.add(new EnclaveAccessMaintainer(controller, intervals.defaultInterval));
- maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer, new SecureRandom()));
+ maintainers.add(new CertificatePoolMaintainer(controller, metric, intervals.certificatePoolMaintainer));
}
public Upgrader upgrader() { return upgrader; }
@@ -197,15 +195,14 @@ public class ControllerMaintenance extends AbstractComponent {
private static class SuccessFactorBaseline {
- private final Double defaultSuccessFactorBaseline;
private final Double deploymentMetricsMaintainerBaseline;
private final Double trafficFractionUpdater;
public SuccessFactorBaseline(SystemName system) {
Objects.requireNonNull(system);
- this.defaultSuccessFactorBaseline = 1.0;
this.deploymentMetricsMaintainerBaseline = 0.90;
this.trafficFractionUpdater = system.isCd() ? 0.5 : 0.65;
}
+
}
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
index edcfcc317a7..a929a1d7af8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java
@@ -24,7 +24,6 @@ import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Set;
import java.util.function.UnaryOperator;
import java.util.logging.Level;
@@ -43,12 +42,10 @@ public class Upgrader extends ControllerMaintainer {
private static final Logger log = Logger.getLogger(Upgrader.class.getName());
private final CuratorDb curator;
- private final Random random;
public Upgrader(Controller controller, Duration interval) {
super(controller, interval);
this.curator = controller.curator();
- this.random = new Random(controller.clock().instant().toEpochMilli()); // Seed with clock for test determinism
}
/**
@@ -78,7 +75,7 @@ public class Upgrader extends ControllerMaintainer {
private InstanceList instances(DeploymentStatusList deploymentStatuses) {
return InstanceList.from(deploymentStatuses)
.withDeclaredJobs()
- .shuffle(random)
+ .shuffle(controller().random(false))
.byIncreasingDeployedVersion()
.unpinned();
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
index bc977baf048..5770649c8b7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
@@ -10,7 +10,9 @@ import com.yahoo.slime.Cursor;
import com.yahoo.slime.Inspector;
import com.yahoo.slime.Slime;
import com.yahoo.slime.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
@@ -45,12 +47,15 @@ public class RoutingPolicySerializer {
private static final String dnsZoneField = "dnsZone";
private static final String instanceEndpointsField = "rotations";
private static final String applicationEndpointsField = "applicationEndpoints";
- private static final String loadBalancerActiveField = "active";
private static final String globalRoutingField = "globalRouting";
private static final String agentField = "agent";
private static final String changedAtField = "changedAt";
private static final String statusField = "status";
private static final String privateOnlyField = "private";
+ private static final String generatedEndpointsField = "generatedEndpoints";
+ private static final String clusterPartField = "clusterPart";
+ private static final String applicationPartField = "applicationPart";
+ private static final String authMethodField = "authMethod";
public Slime toSlime(List<RoutingPolicy> routingPolicies) {
var slime = new Slime();
@@ -69,6 +74,13 @@ public class RoutingPolicySerializer {
policy.applicationEndpoints().forEach(endpointId -> applicationEndpointsArray.addString(endpointId.id()));
globalRoutingToSlime(policy.routingStatus(), policyObject.setObject(globalRoutingField));
if ( ! policy.isPublic()) policyObject.setBool(privateOnlyField, true);
+ Cursor generatedEndpointsArray = policyObject.setArray(generatedEndpointsField);
+ policy.generatedEndpoints().forEach(generatedEndpoint -> {
+ Cursor generatedEndpointObject = generatedEndpointsArray.addObject();
+ generatedEndpointObject.setString(clusterPartField, generatedEndpoint.clusterPart());
+ generatedEndpointObject.setString(applicationPartField, generatedEndpoint.applicationPart());
+ generatedEndpointObject.setString(authMethodField, authMethod(generatedEndpoint.authMethod()));
+ });
});
return slime;
}
@@ -86,6 +98,14 @@ public class RoutingPolicySerializer {
ClusterSpec.Id.from(inspect.field(clusterField).asString()),
ZoneId.from(inspect.field(zoneField).asString()));
boolean isPublic = ! inspect.field(privateOnlyField).asBool();
+ List<GeneratedEndpoint> generatedEndpoints = new ArrayList<>();
+ Inspector generatedEndpointsArray = inspect.field(generatedEndpointsField);
+ if (generatedEndpointsArray.valid()) {
+ generatedEndpointsArray.traverse((ArrayTraverser) (idx, generatedEndpointObject) ->
+ generatedEndpoints.add(new GeneratedEndpoint(generatedEndpointObject.field(clusterPartField).asString(),
+ generatedEndpointObject.field(applicationPartField).asString(),
+ authMethodFromSlime(generatedEndpointObject.field(authMethodField)))));
+ }
policies.add(new RoutingPolicy(id,
SlimeUtils.optionalString(inspect.field(canonicalNameField)).map(DomainName::of),
SlimeUtils.optionalString(inspect.field(ipAddressField)),
@@ -93,7 +113,8 @@ public class RoutingPolicySerializer {
instanceEndpoints,
applicationEndpoints,
routingStatusFromSlime(inspect.field(globalRoutingField)),
- isPublic));
+ isPublic,
+ generatedEndpoints));
});
return Collections.unmodifiableList(policies);
}
@@ -111,4 +132,19 @@ public class RoutingPolicySerializer {
return new RoutingStatus(status, agent, changedAt);
}
+ private String authMethod(Endpoint.AuthMethod authMethod) {
+ return switch (authMethod) {
+ case token -> "token";
+ case mtls -> "mtls";
+ };
+ }
+
+ private Endpoint.AuthMethod authMethodFromSlime(Inspector field) {
+ return switch (field.asString()) {
+ case "token" -> Endpoint.AuthMethod.token;
+ case "mtls" -> Endpoint.AuthMethod.mtls;
+ default -> throw new IllegalArgumentException("Unknown auth method '" + field.asString() + "'");
+ };
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 693275987c5..210b8df1447 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1891,7 +1891,10 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
object.setString("scope", endpointScopeString(endpoint.scope()));
object.setString("routingMethod", routingMethodString(endpoint.routingMethod()));
object.setBool("legacy", endpoint.legacy());
- object.setString("authMethod", endpoint.isTokenEndpoint() ? "token" : "mtls");
+ switch (endpoint.authMethod()) {
+ case mtls -> object.setString("authMethod", "mtls");
+ case token -> object.setString("authMethod", "token");
+ }
}
private void toSlime(Cursor response, DeploymentId deploymentId, Deployment deployment, HttpRequest request) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index a20d5497945..acd39b1c8b3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -22,13 +22,14 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
-import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge;
import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.ChallengeState;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.VpcEndpointService.DnsChallenge;
import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget;
import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedDirectTarget;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder;
import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
@@ -86,7 +87,7 @@ public class RoutingPolicies {
}
/** Read all routing policies for given application */
- private RoutingPolicyList read(TenantAndApplicationId application) {
+ public RoutingPolicyList read(TenantAndApplicationId application) {
return db.readRoutingPolicies((instance) -> TenantAndApplicationId.from(instance).equals(application))
.values()
.stream()
@@ -112,7 +113,7 @@ public class RoutingPolicies {
* Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if
* routing configuration affecting given deployment has changed.
*/
- public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec) {
+ public void refresh(DeploymentId deployment, DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) {
ApplicationId instance = deployment.applicationId();
List<LoadBalancer> loadBalancers = controller.serviceRegistry().configServer()
.getLoadBalancers(instance, deployment.zoneId());
@@ -121,18 +122,17 @@ public class RoutingPolicies {
Optional<TenantAndApplicationId> owner = ownerOf(allocation);
try (var lock = db.lockRoutingPolicies()) {
RoutingPolicyList applicationPolicies = read(TenantAndApplicationId.from(instance));
- RoutingPolicyList instancePolicies = applicationPolicies.instance(instance);
RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(allocation.deployment);
removeGlobalDnsUnreferencedBy(allocation, deploymentPolicies, lock);
removeApplicationDnsUnreferencedBy(allocation, deploymentPolicies, lock);
- instancePolicies = storePoliciesOf(allocation, instancePolicies, lock);
+ RoutingPolicyList instancePolicies = storePoliciesOf(allocation, applicationPolicies, generatedEndpoints, lock);
instancePolicies = removePoliciesUnreferencedBy(allocation, instancePolicies, lock);
- applicationPolicies = applicationPolicies.replace(instance, instancePolicies);
+ RoutingPolicyList updatedApplicationPolicies = applicationPolicies.replace(instance, instancePolicies);
updateGlobalDnsOf(instancePolicies, Optional.of(deployment), inactiveZones, owner, lock);
- updateApplicationDnsOf(applicationPolicies, inactiveZones, deployment, owner, lock);
+ updateApplicationDnsOf(updatedApplicationPolicies, inactiveZones, deployment, owner, lock);
}
}
@@ -363,8 +363,8 @@ public class RoutingPolicies {
*
* @return the updated policies
*/
- private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList instancePolicies, @SuppressWarnings("unused") Mutex lock) {
- Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(instancePolicies.asMap());
+ private RoutingPolicyList storePoliciesOf(LoadBalancerAllocation allocation, RoutingPolicyList applicationPolicies, List<GeneratedEndpoint> generatedEndpoints, @SuppressWarnings("unused") Mutex lock) {
+ Map<RoutingPolicyId, RoutingPolicy> policies = new LinkedHashMap<>(applicationPolicies.instance(allocation.deployment.applicationId()).asMap());
for (LoadBalancer loadBalancer : allocation.loadBalancers) {
if (loadBalancer.hostname().isEmpty() && loadBalancer.ipAddress().isEmpty()) continue;
var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId());
@@ -374,10 +374,17 @@ public class RoutingPolicies {
allocation.instanceEndpointsOf(loadBalancer),
allocation.applicationEndpointsOf(loadBalancer),
RoutingStatus.DEFAULT,
- loadBalancer.isPublic());
- // Preserve global routing status for existing policy
+ loadBalancer.isPublic(),
+ generatedEndpoints);
+ boolean addingGeneratedEndpoints = !generatedEndpoints.isEmpty() && (existingPolicy == null || existingPolicy.generatedEndpoints().isEmpty());
+ if (addingGeneratedEndpoints) {
+ generatedEndpoints.forEach(ge -> requireNonClashing(ge, applicationPolicies));
+ }
if (existingPolicy != null) {
- newPolicy = newPolicy.with(existingPolicy.routingStatus());
+ newPolicy = newPolicy.with(existingPolicy.routingStatus()); // Always preserve routing status
+ if (!addingGeneratedEndpoints) {
+ newPolicy = newPolicy.with(existingPolicy.generatedEndpoints()); // Endpoints are generated once
+ }
}
updateZoneDnsOf(newPolicy, loadBalancer, allocation.deployment);
policies.put(newPolicy.id(), newPolicy);
@@ -402,7 +409,11 @@ public class RoutingPolicies {
private void setPrivateDns(Endpoint endpoint, LoadBalancer loadBalancer, DeploymentId deploymentId) {
if (loadBalancer.service().isEmpty()) return;
- if (endpoint.isTokenEndpoint()) return;
+ boolean skipBasedOnAuthMethod = switch (endpoint.authMethod()) {
+ case token -> true;
+ case mtls -> false;
+ };
+ if (skipBasedOnAuthMethod) return;
controller.serviceRegistry().vpcEndpointService()
.setPrivateDns(DomainName.of(endpoint.dnsName()),
new ClusterId(deploymentId, endpoint.cluster()),
@@ -723,4 +734,14 @@ public class RoutingPolicies {
return ownerOf(allocation.deployment);
}
+ private static void requireNonClashing(GeneratedEndpoint generatedEndpoint, RoutingPolicyList applicationPolicies) {
+ for (var policy : applicationPolicies) {
+ for (var ge : policy.generatedEndpoints()) {
+ if (ge.clusterPart().equals(generatedEndpoint.clusterPart())) {
+ throw new IllegalArgumentException(generatedEndpoint + " clashes with " + ge + " in " + policy.id());
+ }
+ }
+ }
+ }
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index fb8f5e8e129..d25a96b5ace 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -5,12 +5,13 @@ import ai.vespa.http.DomainName;
import com.google.common.collect.ImmutableSortedSet;
import com.yahoo.config.provision.SystemName;
import com.yahoo.config.provision.zone.RoutingMethod;
-import com.yahoo.text.Text;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
+import java.util.ArrayList;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
@@ -29,11 +30,13 @@ public record RoutingPolicy(RoutingPolicyId id,
Set<EndpointId> instanceEndpoints,
Set<EndpointId> applicationEndpoints,
RoutingStatus routingStatus,
- boolean isPublic) {
+ boolean isPublic,
+ List<GeneratedEndpoint> generatedEndpoints) {
/** DO NOT USE. Public for serialization purposes */
public RoutingPolicy(RoutingPolicyId id, Optional<DomainName> canonicalName, Optional<String> ipAddress, Optional<String> dnsZone,
- Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, RoutingStatus routingStatus, boolean isPublic) {
+ Set<EndpointId> instanceEndpoints, Set<EndpointId> applicationEndpoints, RoutingStatus routingStatus, boolean isPublic,
+ List<GeneratedEndpoint> generatedEndpoints) {
this.id = Objects.requireNonNull(id, "id must be non-null");
this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null");
this.ipAddress = Objects.requireNonNull(ipAddress, "ipAddress must be non-null");
@@ -42,6 +45,7 @@ public record RoutingPolicy(RoutingPolicyId id,
this.applicationEndpoints = ImmutableSortedSet.copyOf(Objects.requireNonNull(applicationEndpoints, "applicationEndpoints must be non-null"));
this.routingStatus = Objects.requireNonNull(routingStatus, "status must be non-null");
this.isPublic = isPublic;
+ this.generatedEndpoints = List.copyOf(Objects.requireNonNull(generatedEndpoints, "generatedEndpoints must be non-null"));
if (canonicalName.isEmpty() == ipAddress.isEmpty())
throw new IllegalArgumentException("Exactly 1 of canonicalName=%s and ipAddress=%s must be set".formatted(
@@ -77,11 +81,16 @@ public record RoutingPolicy(RoutingPolicyId id,
return instanceEndpoints;
}
- /** The application-level endpoints this participates in */
+ /** The application-level endpoints this participates in */
public Set<EndpointId> applicationEndpoints() {
return applicationEndpoints;
}
+ /** The endpoints to generate for this policy, if any */
+ public List<GeneratedEndpoint> generatedEndpoints() {
+ return generatedEndpoints;
+ }
+
/** Return status of routing */
public RoutingStatus routingStatus() {
return routingStatus;
@@ -100,19 +109,37 @@ public record RoutingPolicy(RoutingPolicyId id,
/** Returns a copy of this with routing status set to given status */
public RoutingPolicy with(RoutingStatus routingStatus) {
- return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic);
+ return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints);
+ }
+
+ public RoutingPolicy with(List<GeneratedEndpoint> generatedEndpoints) {
+ return new RoutingPolicy(id, canonicalName, ipAddress, dnsZone, instanceEndpoints, applicationEndpoints, routingStatus, isPublic, generatedEndpoints);
}
/** Returns the zone endpoints of this */
public List<Endpoint> zoneEndpointsIn(SystemName system, RoutingMethod routingMethod, boolean includeTokenEndpoint) {
DeploymentId deployment = new DeploymentId(id.owner(), id.zone());
Endpoint zoneEndpoint = endpoint(routingMethod).target(id.cluster(), deployment).in(system);
+ List<Endpoint> endpoints = new ArrayList<>();
+ endpoints.add(zoneEndpoint);
if (includeTokenEndpoint) {
- Endpoint tokenEndpoint = endpoint(routingMethod).target(id.cluster(), deployment).tokenEndpoint().in(system);
- return List.of(zoneEndpoint, tokenEndpoint);
- } else {
- return List.of(zoneEndpoint);
+ Endpoint tokenEndpoint = endpoint(routingMethod).target(id.cluster(), deployment)
+ .authMethod(Endpoint.AuthMethod.token)
+ .in(system);
+ endpoints.add(tokenEndpoint);
+ }
+ for (var generatedEndpoint : generatedEndpoints) {
+ GeneratedEndpoint endpointToInclude = switch (generatedEndpoint.authMethod()) {
+ case token -> includeTokenEndpoint ? generatedEndpoint : null;
+ case mtls -> generatedEndpoint;
+ };
+ if (endpointToInclude != null) {
+ endpoints.add(endpoint(routingMethod).target(id.cluster(), deployment)
+ .generatedEndpoint(endpointToInclude)
+ .in(system));
+ }
}
+ return endpoints;
}
/** Returns the region endpoint of this */
@@ -133,17 +160,10 @@ public record RoutingPolicy(RoutingPolicyId id,
return Objects.hash(id);
}
- @Override
- public String toString() {
- return Text.format("%s [instance endpoints: %s, application endpoints: %s%s], %s owned by %s, in %s", canonicalName,
- instanceEndpoints, applicationEndpoints,
- dnsZone.map(z -> ", DNS zone: " + z).orElse(""), id.cluster(), id.owner().toShortString(),
- id.zone().value());
- }
-
private Endpoint.EndpointBuilder endpoint(RoutingMethod routingMethod) {
return Endpoint.of(id.owner())
.on(Port.fromRoutingMethod(routingMethod))
.routingMethod(routingMethod);
}
+
}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
index 2a7e4cb5c14..2e11a156dce 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java
@@ -11,6 +11,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
@@ -20,7 +21,6 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collectors;
/**
* A deployment routing context, which extends {@link RoutingContext} to support routing configuration of a deployment.
@@ -49,8 +49,8 @@ public abstract class DeploymentRoutingContext implements RoutingContext {
}
/** Configure routing for the deployment in this context, using given deployment spec */
- public final void configure(DeploymentSpec deploymentSpec) {
- controller.policies().refresh(deployment, deploymentSpec);
+ public final void configure(DeploymentSpec deploymentSpec, List<GeneratedEndpoint> generatedEndpoints) {
+ controller.policies().refresh(deployment, deploymentSpec, generatedEndpoints);
}
/** Routing method of this context */
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
index eabbdd76d5a..d9b95a53a0e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java
@@ -66,6 +66,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Optional;
import java.util.OptionalLong;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
@@ -307,11 +308,11 @@ public final class ControllerTester {
}
public TenantName createTenant(String tenantName, Tenant.Type type) {
- switch (type) {
- case athenz: return createTenant(tenantName, "domain" + nextDomainId.getAndIncrement(), nextPropertyId.getAndIncrement());
- case cloud: return createCloudTenant(tenantName);
- default: throw new UnsupportedOperationException();
- }
+ return switch (type) {
+ case athenz -> createTenant(tenantName, "domain" + nextDomainId.getAndIncrement(), nextPropertyId.getAndIncrement());
+ case cloud -> createCloudTenant(tenantName);
+ default -> throw new UnsupportedOperationException();
+ };
}
public TenantName createTenant(String tenantName, String domainName, Long propertyId) {
@@ -347,17 +348,13 @@ public final class ControllerTester {
public Credentials credentialsFor(TenantName tenantName) {
Tenant tenant = controller().tenants().require(tenantName);
- switch (tenant.type()) {
- case athenz:
- return new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")),
- ((AthenzTenant) tenant).domain(),
- OAuthCredentials.createForTesting("okta-access-token", "okta-identity-token"));
- case cloud:
- return new Credentials(new SimplePrincipal("dev"));
-
- default:
- throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'");
- }
+ return switch (tenant.type()) {
+ case athenz -> new AthenzCredentials(new AthenzPrincipal(new AthenzUser("user")),
+ ((AthenzTenant) tenant).domain(),
+ OAuthCredentials.createForTesting("okta-access-token", "okta-identity-token"));
+ case cloud -> new Credentials(new SimplePrincipal("dev"));
+ default -> throw new IllegalArgumentException("Unexpected tenant type '" + tenant.type() + "'");
+ };
}
public Application createApplication(ApplicationId id) {
@@ -385,6 +382,7 @@ public final class ControllerTester {
AthenzDbMock athensDb,
ServiceRegistryMock serviceRegistry,
FlagSource flagSource) {
+ Random random = new Random(serviceRegistry.clock().instant().toEpochMilli()); // Seed with clock for test determinism
Controller controller = new Controller(curator,
rotationsConfig,
serviceRegistry.zoneRegistry().system().isPublic() ?
@@ -395,7 +393,9 @@ public final class ControllerTester {
serviceRegistry,
new MetricsMock(), new SecretStoreMock(),
new ControllerConfig.Builder().build(),
- Sleeper.NOOP);
+ Sleeper.NOOP,
+ random,
+ random);
// Calculate initial versions
controller.updateVersionStatus(VersionStatus.compute(controller));
return controller;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
index dc96aa6c62c..23c029845bb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java
@@ -344,4 +344,36 @@ public class EndpointTest {
tests2.forEach((expected, endpoint) -> assertEquals(expected, endpoint.upstreamName(zone2)));
}
+ @Test
+ public void generated_id() {
+ GeneratedEndpoint ge = new GeneratedEndpoint("cafed00d", "deadbeef", Endpoint.AuthMethod.mtls);
+ var deployment = new DeploymentId(instance1, ZoneId.from("prod", "us-north-1"));
+ var tests = Map.of(
+ // Zone endpoint in main, unlike named endpoints, this includes the scope symbol 'z'
+ "cafed00d.deadbeef.z.vespa.oath.cloud",
+ Endpoint.of(instance1).target(ClusterSpec.Id.from("c1"), deployment).generatedEndpoint(ge)
+ .routingMethod(RoutingMethod.sharedLayer4).on(Port.tls()).in(SystemName.main),
+ // Zone endpoint in public
+ "cafed00d.deadbeef.z.vespa-app.cloud",
+ Endpoint.of(instance1).target(ClusterSpec.Id.from("c1"), deployment).generatedEndpoint(ge)
+ .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public),
+ // Global endpoint in public
+ "foo.deadbeef.g.vespa-app.cloud",
+ Endpoint.of(instance1).target(EndpointId.of("foo"), ClusterSpec.Id.from("c1"), List.of(deployment))
+ .generatedEndpoint(ge)
+ .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public),
+ // Global endpoint in public, with default ID
+ "deadbeef.g.vespa-app.cloud",
+ Endpoint.of(instance1).target(EndpointId.defaultId(), ClusterSpec.Id.from("c1"), List.of(deployment))
+ .generatedEndpoint(ge)
+ .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public),
+ // Application endpoint in public
+ "bar.deadbeef.a.vespa-app.cloud",
+ Endpoint.of(TenantAndApplicationId.from(instance1)).targetApplication(EndpointId.of("bar"), deployment)
+ .generatedEndpoint(ge)
+ .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public)
+ );
+ tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.dnsName()));
+ }
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java
index f94120241e7..a371677b82b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainerTest.java
@@ -10,7 +10,6 @@ import org.junit.jupiter.api.Test;
import java.time.Duration;
import java.util.List;
-import java.util.Random;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -20,7 +19,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
public class CertificatePoolMaintainerTest {
private final ControllerTester tester = new ControllerTester();
- private final CertificatePoolMaintainer maintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1), new Random(4));
+ private final CertificatePoolMaintainer maintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1));
@Test
void new_certs_are_requested_until_limit() {
@@ -41,17 +40,18 @@ public class CertificatePoolMaintainerTest {
assertEquals(
List.of(
- new DnsNameStatus("*.c8868d4e.z.vespa.oath.cloud", "done"),
- new DnsNameStatus("*.c8868d4e.g.vespa.oath.cloud", "done"),
- new DnsNameStatus("*.c8868d4e.a.vespa.oath.cloud", "done")
+ new DnsNameStatus("*.f5549014.z.vespa.oath.cloud", "done"),
+ new DnsNameStatus("*.f5549014.g.vespa.oath.cloud", "done"),
+ new DnsNameStatus("*.f5549014.a.vespa.oath.cloud", "done")
), metadata.dnsNames());
- assertEquals("vespa.tls.preprovisioned.c8868d4e-cert", endpointCertificateProvider.certificateDetails(metadata.requestId()).cert_key_keyname());
- assertEquals("vespa.tls.preprovisioned.c8868d4e-key", endpointCertificateProvider.certificateDetails(metadata.requestId()).private_key_keyname());
+ assertEquals("vespa.tls.preprovisioned.f5549014-cert", endpointCertificateProvider.certificateDetails(metadata.requestId()).cert_key_keyname());
+ assertEquals("vespa.tls.preprovisioned.f5549014-key", endpointCertificateProvider.certificateDetails(metadata.requestId()).private_key_keyname());
}
private void assertNumCerts(int n) {
assertEquals(0.0, maintainer.maintain(), 0.0000001);
assertEquals(n, tester.curator().readUnassignedCertificates().size());
}
+
}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
index 24eb9f33d33..247ffe1de00 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -31,7 +31,6 @@ import java.time.temporal.ChronoUnit;
import java.util.List;
import java.util.Optional;
import java.util.OptionalDouble;
-import java.util.Random;
import java.util.stream.Stream;
import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1;
@@ -49,7 +48,7 @@ public class EndpointCertificateMaintainerTest {
private final ControllerTester tester = new ControllerTester();
private final SecretStoreMock secretStore = (SecretStoreMock) tester.controller().secretStore();
private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1));
- private final CertificatePoolMaintainer certificatePoolMaintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1), new Random(4));
+ private final CertificatePoolMaintainer certificatePoolMaintainer = new CertificatePoolMaintainer(tester.controller(), new MockMetric(), Duration.ofHours(1));
private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty());
@Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
index c1267ad5edf..f685c75bbe3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
@@ -5,7 +5,9 @@ import com.yahoo.config.provision.ApplicationId;
import com.yahoo.config.provision.ClusterSpec;
import com.yahoo.config.provision.HostName;
import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
@@ -44,7 +46,8 @@ public class RoutingPolicySerializerTest {
Set.of(),
Set.of(),
RoutingStatus.DEFAULT,
- false),
+ false,
+ List.of(new GeneratedEndpoint("deadbeef", "cafed00d", Endpoint.AuthMethod.mtls))),
new RoutingPolicy(id2,
Optional.of(HostName.of("long-and-ugly-name-2")),
Optional.empty(),
@@ -54,7 +57,8 @@ public class RoutingPolicySerializerTest {
new RoutingStatus(RoutingStatus.Value.out,
RoutingStatus.Agent.tenant,
Instant.ofEpochSecond(123)),
- true),
+ true,
+ List.of(new GeneratedEndpoint("cafed00d", "deadbeef", Endpoint.AuthMethod.token))),
new RoutingPolicy(id1,
Optional.empty(),
Optional.of("127.0.0.1"),
@@ -62,7 +66,8 @@ public class RoutingPolicySerializerTest {
instanceEndpoints,
applicationEndpoints,
RoutingStatus.DEFAULT,
- true));
+ true,
+ List.of()));
var serialized = serializer.fromSlime(owner, serializer.toSlime(policies));
assertEquals(policies.size(), serialized.size());
for (Iterator<RoutingPolicy> it1 = policies.iterator(), it2 = serialized.iterator(); it1.hasNext(); ) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 0233db50ac6..783629c8f4a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.hosted.controller.routing;
import ai.vespa.http.DomainName;
import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Sets;
import com.yahoo.config.application.api.DeploymentSpec;
import com.yahoo.config.application.api.ValidationId;
import com.yahoo.config.provision.ApplicationId;
@@ -20,6 +19,7 @@ import com.yahoo.vespa.flags.Flags;
import com.yahoo.vespa.hosted.controller.ControllerTester;
import com.yahoo.vespa.hosted.controller.Instance;
import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
import com.yahoo.vespa.hosted.controller.api.integration.dns.Record.Type;
@@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.EndpointList;
import com.yahoo.vespa.hosted.controller.application.SystemApplication;
import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate;
import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
@@ -53,7 +54,6 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.stream.Collector;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -262,11 +262,11 @@ public class RoutingPoliciesTest {
context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
// Deployment creates records and policies for all clusters in all zones
- Set<String> expectedRecords = Set.of(
- "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+ List<String> expectedRecords = List.of(
"c0.app1.tenant1.us-central-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-central-1.vespa.oath.cloud"
+ "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
assertEquals(4, tester.policiesOf(context1.instanceId()).size());
@@ -279,13 +279,13 @@ public class RoutingPoliciesTest {
// Add 1 cluster in each zone and deploy
tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2);
context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
- expectedRecords = Set.of(
- "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c2.app1.tenant1.us-west-1.vespa.oath.cloud",
+ expectedRecords = List.of(
"c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
"c1.app1.tenant1.us-central-1.vespa.oath.cloud",
- "c2.app1.tenant1.us-central-1.vespa.oath.cloud"
+ "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+ "c2.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c2.app1.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
assertEquals(6, tester.policiesOf(context1.instanceId()).size());
@@ -293,17 +293,17 @@ public class RoutingPoliciesTest {
// Deploy another application
tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2);
context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
- expectedRecords = Set.of(
- "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c2.app1.tenant1.us-west-1.vespa.oath.cloud",
+ expectedRecords = List.of(
"c0.app1.tenant1.us-central-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
- "c2.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
"c0.app2.tenant1.us-central-1.vespa.oath.cloud",
- "c1.app2.tenant1.us-central-1.vespa.oath.cloud",
"c0.app2.tenant1.us-west-1.vespa.oath.cloud",
- "c1.app2.tenant1.us-west-1.vespa.oath.cloud"
+ "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+ "c1.app2.tenant1.us-central-1.vespa.oath.cloud",
+ "c1.app2.tenant1.us-west-1.vespa.oath.cloud",
+ "c2.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c2.app1.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords.stream().sorted().toList(), tester.recordNames().stream().sorted().toList());
assertEquals(4, tester.policiesOf(context2.instanceId()).size());
@@ -311,14 +311,14 @@ public class RoutingPoliciesTest {
// Deploy removes cluster from app1
tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2);
context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
- expectedRecords = Set.of(
- "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+ expectedRecords = List.of(
"c0.app1.tenant1.us-central-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
"c0.app2.tenant1.us-central-1.vespa.oath.cloud",
- "c1.app2.tenant1.us-central-1.vespa.oath.cloud",
"c0.app2.tenant1.us-west-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+ "c1.app2.tenant1.us-central-1.vespa.oath.cloud",
"c1.app2.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
@@ -327,11 +327,11 @@ public class RoutingPoliciesTest {
tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet()
.forEach(zone -> tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone));
context2.flushDnsUpdates();
- expectedRecords = Set.of(
- "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+ expectedRecords = List.of(
"c0.app1.tenant1.us-central-1.vespa.oath.cloud",
- "c1.app1.tenant1.us-central-1.vespa.oath.cloud"
+ "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "c1.app1.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
assertTrue(tester.routingPolicies().read(context2.instanceId()).isEmpty(), "Removes stale routing policies " + context2.application());
@@ -350,11 +350,11 @@ public class RoutingPoliciesTest {
context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
// Deployment creates records and policies for all clusters in all zones
- Set<String> expectedRecords = Set.of(
- "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
- "token-c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+ List<String> expectedRecords = List.of(
"c0.app1.tenant1.us-central-1.vespa.oath.cloud",
- "token-c0.app1.tenant1.us-central-1.vespa.oath.cloud"
+ "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+ "token-c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+ "token-c0.app1.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
assertEquals(2, tester.policiesOf(context1.instanceId()).size());
@@ -367,7 +367,7 @@ public class RoutingPoliciesTest {
tester.provisionLoadBalancers(1, context.instanceId(), true, zone1, zone2);
context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
assertEquals(0, tester.controllerTester().controller().curator().readNameServiceQueue().requests().size());
- assertEquals(Set.of(), tester.recordNames());
+ assertEquals(List.of(), tester.recordNames());
assertEquals(2, tester.policiesOf(context.instanceId()).size());
}
@@ -409,15 +409,17 @@ public class RoutingPoliciesTest {
.build();
context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
- List<String> expectedRecords = List.of("c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
- "c0.app1.tenant1.gcp-us-south1-b.z.vespa-app.cloud",
- "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud",
- "c0.app1.tenant1.gcp-us-south1.w.vespa-app.cloud",
- "r0.app1.tenant1.g.vespa-app.cloud");
- assertEquals(Set.copyOf(expectedRecords), tester.recordNames());
+ List<String> expectedRecords = List.of(
+ "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud",
+ "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+ "c0.app1.tenant1.gcp-us-south1-b.z.vespa-app.cloud",
+ "c0.app1.tenant1.gcp-us-south1.w.vespa-app.cloud",
+ "r0.app1.tenant1.g.vespa-app.cloud"
+ );
+ assertEquals(expectedRecords, tester.recordNames());
- assertEquals(List.of("lb-0--tenant1.app1.default--prod.aws-us-east-1c."), tester.recordDataOf(Record.Type.CNAME, expectedRecords.get(0)));
- assertEquals(List.of("10.0.0.0"), tester.recordDataOf(Record.Type.A, expectedRecords.get(1)));
+ assertEquals(List.of("lb-0--tenant1.app1.default--prod.aws-us-east-1c."), tester.recordDataOf(Record.Type.CNAME, expectedRecords.get(1)));
+ assertEquals(List.of("10.0.0.0"), tester.recordDataOf(Record.Type.A, expectedRecords.get(2)));
assertEquals(List.of("weighted/10.0.0.0/prod.gcp-us-south1-b/1"), tester.recordDataOf(Record.Type.DIRECT, expectedRecords.get(3)));
assertEquals(List.of("latency/c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud/dns-zone-1/prod.aws-us-east-1c",
"latency/c0.app1.tenant1.gcp-us-south1.w.vespa-app.cloud/ignored/prod.gcp-us-south1-b"),
@@ -443,11 +445,12 @@ public class RoutingPoliciesTest {
tester.assertTargets(context.instanceId(), EndpointId.defaultId(),
ClusterSpec.Id.from("default"), 0,
Map.of(zone1, 1L, zone2, 1L));
- assertEquals(Set.of("app1.tenant1.aws-eu-west-1.w.vespa-app.cloud",
- "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
- "app1.tenant1.aws-us-east-1.w.vespa-app.cloud",
- "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
- "app1.tenant1.g.vespa-app.cloud"),
+ assertEquals(List.of("app1.tenant1.aws-eu-west-1.w.vespa-app.cloud",
+ "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+ "app1.tenant1.aws-us-east-1.w.vespa-app.cloud",
+ "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+ "app1.tenant1.g.vespa-app.cloud"
+ ),
tester.recordNames(),
"Registers expected DNS names");
}
@@ -471,7 +474,7 @@ public class RoutingPoliciesTest {
// Routing policy is created and DNS is updated
assertEquals(1, tester.policiesOf(context.instanceId()).size());
- assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
+ assertEquals(List.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
}
@Test
@@ -482,7 +485,7 @@ public class RoutingPoliciesTest {
context.submit(applicationPackage).deploy();
var zone = ZoneId.from("dev", "us-east-1");
tester.controllerTester().setRoutingMethod(List.of(zone), RoutingMethod.exclusive);
- var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud");
+ var prodRecords = List.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud");
assertEquals(prodRecords, tester.recordNames());
// Deploy to dev under different instance
@@ -494,7 +497,8 @@ public class RoutingPoliciesTest {
// Routing policy is created and DNS is updated
assertEquals(1, tester.policiesOf(devContext.instanceId()).size());
- assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames());
+ assertEquals(Stream.concat(prodRecords.stream(), Stream.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")).sorted().toList(),
+ tester.recordNames());
}
@Test
@@ -510,7 +514,7 @@ public class RoutingPoliciesTest {
// Application is deployed
context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
- var expectedRecords = Set.of(
+ var expectedRecords = List.of(
"c0.app1.tenant1.us-west-1.vespa.oath.cloud"
);
assertEquals(expectedRecords, tester.recordNames());
@@ -557,8 +561,8 @@ public class RoutingPoliciesTest {
app.deploy();
// TXT records are cleaned up as we go—the last challenge is the last to go here, and we must flush it ourselves.
- assertEquals(Set.of("a.t.aws-us-east-1a.vespa.oath.cloud",
- "challenge--a.t.aws-us-east-1a.vespa.oath.cloud"),
+ assertEquals(List.of("a.t.aws-us-east-1a.vespa.oath.cloud",
+ "challenge--a.t.aws-us-east-1a.vespa.oath.cloud"),
tester.recordNames());
app.flushDnsUpdates();
assertEquals(Set.of(new Record(Type.CNAME,
@@ -773,7 +777,7 @@ public class RoutingPoliciesTest {
tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out,
RoutingStatus.Agent.tenant);
} catch (IllegalArgumentException e) {
- assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive]", e.getMessage());
+ assertEquals("Cannot deactivate routing for tenant1.app1 in prod.us-central-1 as it's the last remaining active deployment in endpoint https://r0.app1.tenant1.global.vespa.oath.cloud/ [scope=global, legacy=false, routingMethod=exclusive, authMethod=mtls]", e.getMessage());
}
context.flushDnsUpdates();
tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
@@ -841,9 +845,9 @@ public class RoutingPoliciesTest {
// Application endpoints are not created until production jobs run
betaContext.submit(applicationPackage)
.runJob(DeploymentContext.systemTest);
- assertEquals(Set.of("beta.app1.tenant1.us-east-1.test.vespa.oath.cloud"), tester.recordNames());
+ assertEquals(List.of("beta.app1.tenant1.us-east-1.test.vespa.oath.cloud"), tester.recordNames());
betaContext.runJob(DeploymentContext.stagingTest);
- assertEquals(Set.of("beta.app1.tenant1.us-east-3.staging.vespa.oath.cloud"), tester.recordNames());
+ assertEquals(List.of("beta.app1.tenant1.us-east-3.staging.vespa.oath.cloud"), tester.recordNames());
// Deploy both instances
betaContext.completeRollout();
@@ -958,7 +962,7 @@ public class RoutingPoliciesTest {
tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
fail("Expected exception");
} catch (IllegalArgumentException e) {
- assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive]",
+ assertEquals("Cannot deactivate routing for tenant1.app1.main in prod.south as it's the last remaining active deployment in endpoint https://a0.app1.tenant1.a.vespa.oath.cloud/ [scope=application, legacy=false, routingMethod=exclusive, authMethod=mtls]",
e.getMessage());
}
@@ -1004,6 +1008,53 @@ public class RoutingPoliciesTest {
"Policies removed");
}
+ @Test
+ public void generated_zone_endpoints() {
+ var tester = new RoutingPoliciesTester(SystemName.Public);
+ var context = tester.newDeploymentContext("tenant1", "app1", "default");
+ tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true);
+ addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester);
+
+ // Deploy application
+ int clustersPerZone = 1;
+ var zone1 = ZoneId.from("prod", "aws-us-east-1c");
+ var zone2 = ZoneId.from("prod", "aws-eu-west-1a");
+ ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
+ .region(zone2.region())
+ .build();
+ tester.provisionLoadBalancers(clustersPerZone, context.instanceId(), zone1, zone2);
+ context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+ // Deployment creates generated zone names
+ List<String> expectedRecords = List.of(
+ "a9c8c045.cafed00d.z.vespa-app.cloud",
+ "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+ "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+ "e144a11b.cafed00d.z.vespa-app.cloud"
+ );
+ assertEquals(expectedRecords, tester.recordNames());
+ assertEquals(2, tester.policiesOf(context.instanceId()).size());
+ for (var zone : List.of(zone1, zone2)) {
+ EndpointList endpoints = tester.controllerTester().controller().routing().readEndpointsOf(context.deploymentIdIn(zone));
+ assertEquals(1, endpoints.generated().size());
+ }
+
+ // Next deployment does not change generated names
+ context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+ assertEquals(expectedRecords, tester.recordNames());
+ }
+
+ private void addCertificateToPool(String id, UnassignedCertificate.State state, RoutingPoliciesTester tester) {
+ EndpointCertificateMetadata cert = new EndpointCertificateMetadata("testKey", "testCert", 1, 0,
+ "request-id",
+ Optional.of("leaf-request-uuid"),
+ List.of("name1", "name2"),
+ "", Optional.empty(),
+ Optional.empty(), Optional.of(id));
+ UnassignedCertificate pooledCert = new UnassignedCertificate(cert, state);
+ tester.controllerTester().controller().curator().writeUnassignedCertificate(pooledCert);
+ }
+
/** Returns an application package builder that satisfies requirements for a directly routed endpoint */
private static ApplicationPackageBuilder applicationPackageBuilder() {
return new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"),
@@ -1113,11 +1164,13 @@ public class RoutingPoliciesTest {
return tester.controller().routing().policies().read(instance);
}
- private Set<String> recordNames() {
+ private List<String> recordNames() {
return tester.controllerTester().nameService().records().stream()
.map(Record::name)
.map(RecordName::asString)
- .collect(Collectors.toSet());
+ .distinct()
+ .sorted()
+ .toList();
}
private Set<String> aliasDataOf(String name) {