diff options
author | Martin Polden <mpolden@mpolden.no> | 2023-10-12 09:46:05 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-10-12 09:46:05 +0200 |
commit | 0f14059f373b486987f82b1d161ff89b01f03204 (patch) | |
tree | ad68405fffdc92e9af68739ebef593f5fd363f95 /controller-server/src/main/java/com | |
parent | e9562b0af56cf1163e41c99f30c2c836aa1720f1 (diff) | |
parent | 2f6bcf34688f229529e25e2f09d0552c3214318d (diff) |
Merge pull request #28842 from vespa-engine/mpolden/cert-assignment
Refactor certificate assignment and migration
Diffstat (limited to 'controller-server/src/main/java/com')
14 files changed, 375 insertions, 323 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 50718429a2b..0de0ea06904 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -524,7 +524,7 @@ public class ApplicationController { try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); - return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger); + return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger, lock); } }; @@ -569,13 +569,16 @@ public class ApplicationController { private PreparedEndpoints prepareEndpoints(DeploymentId deployment, JobId job, LockedApplication application, ApplicationPackageStream applicationPackage, - Consumer<String> deployLogger) { + Consumer<String> deployLogger, + Mutex applicationLock) { Instance instance = application.get().require(job.application().instance()); Tags tags = applicationPackage.truncatedPackage().deploymentSpec().instance(instance.name()) .map(DeploymentInstanceSpec::tags) .orElseGet(Tags::empty); - Optional<EndpointCertificate> certificate = endpointCertificates.get(instance, deployment.zoneId(), applicationPackage.truncatedPackage().deploymentSpec()); - certificate.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); + EndpointCertificate certificate = endpointCertificates.get(deployment, + applicationPackage.truncatedPackage().deploymentSpec(), + applicationLock); + deployLogger.accept("Using CA signed certificate version %s".formatted(certificate.version())); BasicServicesXml services = applicationPackage.truncatedPackage().services(deployment, tags); return controller.routing().of(deployment).prepare(services, certificate, application); } @@ -696,7 +699,7 @@ public class ApplicationController { if (preparedEndpoints == null) return DeploymentEndpoints.none; PreparedEndpoints prepared = preparedEndpoints.get(); generatedEndpoints.set(prepared.endpoints().generated()); - return new DeploymentEndpoints(prepared.containerEndpoints(), prepared.certificate()); + return new DeploymentEndpoints(prepared.containerEndpoints(), Optional.of(prepared.certificate())); }; Supplier<List<DataplaneTokenVersions>> dataplaneTokenVersions = () -> { Tags tags = applicationPackage.truncatedPackage().deploymentSpec() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index 98d8feda0bb..51e20d0017c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpointList; import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; import com.yahoo.vespa.hosted.controller.routing.RoutingId; @@ -121,8 +122,21 @@ public class RoutingController { return rotationRepository; } + /** Returns the endpoint config to use for given instance */ + public EndpointConfig endpointConfig(ApplicationId instance) { + // TODO(mpolden): Switch to reading endpoint-config flag + if (legacyEndpointsEnabled(instance)) { + if (generatedEndpointsEnabled(instance)) { + return EndpointConfig.combined; + } else { + return EndpointConfig.legacy; + } + } + return EndpointConfig.generated; + } + /** Prepares and returns the endpoints relevant for given deployment */ - public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) { + public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { EndpointList endpoints = EndpointList.EMPTY; DeploymentSpec spec = application.get().deploymentSpec(); @@ -136,7 +150,7 @@ public class RoutingController { // Add zone-scoped endpoints Map<EndpointId, List<GeneratedEndpoint>> generatedForDeclaredEndpoints = new HashMap<>(); Set<ClusterSpec.Id> clustersWithToken = new HashSet<>(); - boolean generatedEndpointsEnabled = generatedEndpointsEnabled(deployment.applicationId()); + EndpointConfig config = endpointConfig(deployment.applicationId()); RoutingPolicyList applicationPolicies = policies().read(TenantAndApplicationId.from(deployment.applicationId())); RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(deployment); for (var container : services.containers()) { @@ -153,7 +167,7 @@ public class RoutingController { if (tokenSupported) { generatedForCluster = generateEndpoints(AuthMethod.token, certificate, Optional.empty(), generatedForCluster); } - GeneratedEndpointList generatedEndpoints = generatedEndpointsEnabled ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; + GeneratedEndpointList generatedEndpoints = config.supportsGenerated() ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedEndpoints).scope(Scope.zone)); } @@ -185,7 +199,7 @@ public class RoutingController { return generatedEndpoints; }); }); - Map<EndpointId, GeneratedEndpointList> generatedEndpoints = generatedEndpointsEnabled + Map<EndpointId, GeneratedEndpointList> generatedEndpoints = config.supportsGenerated() ? generatedForDeclaredEndpoints.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, kv -> GeneratedEndpointList.copyOf(kv.getValue()))) @@ -380,7 +394,24 @@ public class RoutingController { } /** Returns certificate DNS names (CN and SAN values) for given deployment */ - public List<String> certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { + public List<String> certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec, String generatedId, boolean legacy) { + List<String> endpointDnsNames = new ArrayList<>(); + if (legacy) { + endpointDnsNames.addAll(legacyCertificateDnsNames(deployment, deploymentSpec)); + } + for (Scope scope : List.of(Scope.zone, Scope.global, Scope.application)) { + endpointDnsNames.add(Endpoint.of(deployment.applicationId()) + .wildcardGenerated(generatedId, scope) + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .certificateName() + .in(controller.system()) + .dnsName()); + } + return Collections.unmodifiableList(endpointDnsNames); + } + + private List<String> legacyCertificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { List<String> endpointDnsNames = new ArrayList<>(); // We add first an endpoint name based on a hash of the application ID, @@ -447,10 +478,7 @@ public class RoutingController { } private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { - if (generatedEndpointsEnabled(instance) && !legacyEndpointsEnabled(instance)) { - return endpoints.generated(); - } - return endpoints; + return endpointConfig(instance) == EndpointConfig.generated ? endpoints.generated() : endpoints; } private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { @@ -491,13 +519,13 @@ public class RoutingController { } /** Returns generated endpoints. A new endpoint is generated if no matching endpoint already exists */ - private List<GeneratedEndpoint> generateEndpoints(AuthMethod authMethod, Optional<EndpointCertificate> certificate, + private List<GeneratedEndpoint> generateEndpoints(AuthMethod authMethod, EndpointCertificate certificate, Optional<EndpointId> declaredEndpoint, List<GeneratedEndpoint> current) { if (current.stream().anyMatch(e -> e.authMethod() == authMethod && e.endpoint().equals(declaredEndpoint))) { return current; } - Optional<String> applicationPart = certificate.flatMap(EndpointCertificate::generatedId); + Optional<String> applicationPart = certificate.generatedId(); if (applicationPart.isPresent()) { current = new ArrayList<>(current); current.add(new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), @@ -572,14 +600,14 @@ public class RoutingController { return Collections.unmodifiableList(routingMethods); } - public boolean generatedEndpointsEnabled(ApplicationId instance) { + private boolean generatedEndpointsEnabled(ApplicationId instance) { return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) .value(); } - public boolean legacyEndpointsEnabled(ApplicationId instance) { + private boolean legacyEndpointsEnabled(ApplicationId instance) { return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java index d95bb0f9f1b..39e1c89c202 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java @@ -65,7 +65,7 @@ public class Endpoint { Objects.requireNonNull(generated, "generated must be non-null"); this.id = requireEndpointId(id, scope, certificateName); this.cluster = requireCluster(cluster, certificateName); - this.instance = requireInstance(instanceName, scope); + this.instance = requireInstance(instanceName, scope, certificateName, generated.isPresent()); this.url = url; this.targets = List.copyOf(requireTargets(targets, application, instanceName, scope, certificateName)); this.scope = requireScope(scope, routingMethod); @@ -259,7 +259,7 @@ public class Endpoint { } /** Returns the DNS suffix used for endpoints in given system */ - public static String dnsSuffix(SystemName system) { + private static String dnsSuffix(SystemName system) { return switch (system) { case cd -> CD_OATH_DNS_SUFFIX; case main -> MAIN_OATH_DNS_SUFFIX; @@ -316,7 +316,10 @@ public class Endpoint { return endpointId; } - private static Optional<InstanceName> requireInstance(Optional<InstanceName> instanceName, Scope scope) { + private static Optional<InstanceName> requireInstance(Optional<InstanceName> instanceName, Scope scope, boolean certificateName, boolean generated) { + if (generated && certificateName) { + return instanceName; + } if (scope == Scope.application) { if (instanceName.isPresent()) throw new IllegalArgumentException("Instance cannot be set for scope " + scope); } else { @@ -331,7 +334,8 @@ public class Endpoint { } private static List<Target> requireTargets(List<Target> targets, TenantAndApplicationId application, Optional<InstanceName> instanceName, Scope scope, boolean certificateName) { - if (!certificateName && targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); + if (certificateName && targets.isEmpty()) return List.of(); + if (targets.isEmpty()) throw new IllegalArgumentException("At least one target must be given for " + scope + " endpoints"); if (scope == Scope.zone && targets.size() != 1) throw new IllegalArgumentException("Exactly one target must be given for " + scope + " endpoints"); for (var target : targets) { if (scope == Scope.application) { @@ -524,6 +528,18 @@ public class Endpoint { return target(ClusterSpec.Id.from("*"), deployment); } + /** Sets the generated wildcard target for this */ + public EndpointBuilder wildcardGenerated(String applicationPart, Scope scope) { + this.cluster = ClusterSpec.Id.from("*"); + if (scope.multiDeployment()) { + this.endpointId = EndpointId.of("*"); + } + this.targets = List.of(); + this.scope = requireUnset(scope); + this.generated = Optional.of(new GeneratedEndpoint("*", applicationPart, AuthMethod.mtls, Optional.ofNullable(endpointId))); + return this; + } + /** Sets the application target with given ID, cluster, deployments and their weights */ public EndpointBuilder targetApplication(EndpointId endpointId, ClusterSpec.Id cluster, Map<DeploymentId, Integer> deployments) { this.endpointId = endpointId; diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java index b7929240d76..5f75d6105b5 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GeneratedEndpoint.java @@ -17,7 +17,8 @@ import java.util.regex.Pattern; */ public record GeneratedEndpoint(String clusterPart, String applicationPart, AuthMethod authMethod, Optional<EndpointId> endpoint) { - private static final Pattern PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$"); + private static final Pattern CLUSTER_PART_PATTERN = Pattern.compile("^([a-f][a-f0-9]{7}|\\*)$"); + private static final Pattern APPLICATION_PART_PATTERN = Pattern.compile("^[a-f][a-f0-9]{7}$"); public GeneratedEndpoint { Objects.requireNonNull(clusterPart); @@ -25,8 +26,8 @@ public record GeneratedEndpoint(String clusterPart, String applicationPart, Auth Objects.requireNonNull(authMethod); Objects.requireNonNull(endpoint); - Validation.requireMatch(clusterPart, "Cluster part", PART_PATTERN); - Validation.requireMatch(applicationPart, "Application part", PART_PATTERN); + Validation.requireMatch(clusterPart, "Cluster part", CLUSTER_PART_PATTERN); + Validation.requireMatch(applicationPart, "Application part", APPLICATION_PART_PATTERN); } /** Returns whether this was generated for an endpoint declared in {@link com.yahoo.config.application.api.DeploymentSpec} */ diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java index 18b537efd8c..49e2dc5bb0d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java @@ -15,10 +15,19 @@ import java.util.Optional; */ public record AssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instance, - EndpointCertificate certificate) { + EndpointCertificate certificate, + boolean shouldValidate) { public AssignedCertificate with(EndpointCertificate certificate) { - return new AssignedCertificate(application, instance, certificate); + return new AssignedCertificate(application, instance, certificate, shouldValidate); + } + + public AssignedCertificate withoutInstance() { + return new AssignedCertificate(application, Optional.empty(), certificate, shouldValidate); + } + + public AssignedCertificate withShouldValidate(boolean shouldValidate) { + return new AssignedCertificate(application, instance, certificate, shouldValidate); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index 9f03e3f0072..391c9806f0a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -11,18 +11,18 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidator; import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import java.time.Clock; import java.time.Duration; @@ -30,26 +30,29 @@ import java.time.Instant; import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate.State; /** - * Looks up stored endpoint certificate, provisions new certificates if none is found, - * and re-provisions the certificate if the deploying-to zone is not covered. + * This provisions, assigns and updates the certificate for a given deployment. * * See also {@link com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer}, which handles * refreshes, deletions and triggers deployments. * * @author andreer + * @author mpolden */ public class EndpointCertificates { - private static final Logger log = Logger.getLogger(EndpointCertificates.class.getName()); + private static final Logger LOG = Logger.getLogger(EndpointCertificates.class.getName()); + private static final Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time private final Controller controller; private final CuratorDb curator; @@ -58,150 +61,216 @@ public class EndpointCertificates { private final EndpointCertificateValidator certificateValidator; private final BooleanFlag useAlternateCertProvider; private final StringFlag endpointCertificateAlgo; - private final BooleanFlag assignLegacyNames; - private final static Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, EndpointCertificateValidator certificateValidator) { this.controller = controller; this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignLegacyNames = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); this.curator = controller.curator(); this.clock = controller.clock(); this.certificateProvider = certificateProvider; this.certificateValidator = certificateValidator; } - /** Returns a suitable certificate for endpoints of given instance and zone */ - public Optional<EndpointCertificate> get(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { + /** Returns a suitable certificate for endpoints of given deployment */ + public EndpointCertificate get(DeploymentId deployment, DeploymentSpec deploymentSpec, Mutex applicationLock) { + Objects.requireNonNull(applicationLock); Instant start = clock.instant(); - Optional<EndpointCertificate> cert = getOrProvision(instance, zone, deploymentSpec); + EndpointConfig config = controller.routing().endpointConfig(deployment.applicationId()); + EndpointCertificate certificate = assignTo(deployment, deploymentSpec, config); Duration duration = Duration.between(start, clock.instant()); - if (duration.toSeconds() > 30) - log.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); - - if (controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(zone)) { // Until CKMS is available from GCP - if (cert.isPresent()) { - // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time - certificateValidator.validate(cert.get(), instance.id().serializedForm(), zone, controller.routing().certificateDnsNames(new DeploymentId(instance.id(), zone), deploymentSpec)); - GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); - String mangledCertName = "endpointCert_" + cert.get().certName().replace('.', '_') + "-v" + cert.get().version(); // Google cloud does not accept dots in secrets, but they accept underscores - String mangledKeyName = "endpointCert_" + cert.get().keyName().replace('.', '_') + "-v" + cert.get().version(); // Google cloud does not accept dots in secrets, but they accept underscores - if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { - gcpSecretStore.setSecret(mangledCertName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledCertName, - controller.secretStore().getSecret(cert.get().certName(), cert.get().version())); - } - if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { - gcpSecretStore.setSecret(mangledKeyName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledKeyName, - controller.secretStore().getSecret(cert.get().keyName(), cert.get().version())); - } - - return Optional.of(cert.get().withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName)); - } + if (duration.toSeconds() > 30) { + LOG.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", deployment.applicationId().serializedForm(), duration.toSeconds())); } + if (isGcp(deployment)) { + // This is needed until CKMS is available from GCP + return validateGcpCertificate(deployment, deploymentSpec, certificate, config); + } + return certificate; + } - return cert; + private boolean isGcp(DeploymentId deployment) { + return controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(deployment.zoneId()); } - private EndpointCertificate assignFromPool(Instance instance, ZoneId zone) { - // For deployments to manually deployed environments: use per instance certificate - // For all other environments (apply in order): - // * Use per instance certificate if it exists and is assigned a randomized id - // * Use per application certificate if it exits and is assigned a randomized id - // * Assign from pool - - TenantAndApplicationId application = TenantAndApplicationId.from(instance.id()); - Optional<AssignedCertificate> perInstanceAssignedCertificate = curator.readAssignedCertificate(application, Optional.of(instance.name())); - if (perInstanceAssignedCertificate.isPresent() && perInstanceAssignedCertificate.get().certificate().generatedId().isPresent()) { - return updateLastRequested(perInstanceAssignedCertificate.get()).certificate(); - } else if (! zone.environment().isManuallyDeployed()) { - Optional<AssignedCertificate> perApplicationAssignedCertificate = curator.readAssignedCertificate(application, Optional.empty()); - if (perApplicationAssignedCertificate.isPresent() && perApplicationAssignedCertificate.get().certificate().generatedId().isPresent()) { - return updateLastRequested(perApplicationAssignedCertificate.get()).certificate(); - } + private EndpointCertificate validateGcpCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointCertificate certificate, EndpointConfig config) { + // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time + List<String> dnsNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, certificate.generatedId().get(), config.supportsLegacy()); + certificateValidator.validate(certificate, deployment.applicationId().serializedForm(), deployment.zoneId(), dnsNames); + GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); + String mangledCertName = "endpointCert_" + certificate.certName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores + String mangledKeyName = "endpointCert_" + certificate.keyName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores + if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { + gcpSecretStore.setSecret(mangledCertName, + Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), + "endpoint-cert-accessor"); + gcpSecretStore.addSecretVersion(mangledCertName, + controller.secretStore().getSecret(certificate.certName(), certificate.version())); + } + if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { + gcpSecretStore.setSecret(mangledKeyName, + Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), + "endpoint-cert-accessor"); + gcpSecretStore.addSecretVersion(mangledKeyName, + controller.secretStore().getSecret(certificate.keyName(), certificate.version())); } + return certificate.withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName); + } - // For new applications which is assigned from pool we follow these rules: - // Assign certificate per instance only in manually deployed environments. In other environments, we share the - // certificate because application endpoints can span instances - Optional<InstanceName> instanceName = zone.environment().isManuallyDeployed() ? Optional.of(instance.name()) : Optional.empty(); + private AssignedCertificate assignFromPool(TenantAndApplicationId application, Optional<InstanceName> instanceName, ZoneId zone) { try (Mutex lock = controller.curator().lockCertificatePool()) { Optional<UnassignedCertificate> candidate = curator.readUnassignedCertificates().stream() .filter(pc -> pc.state() == State.ready) .min(Comparator.comparingLong(pc -> pc.certificate().lastRequested())); if (candidate.isEmpty()) { - throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + instance.id() + " in " + zone); + throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + + application + instanceName.map(i -> "." + i.value()).orElse("") + + " in " + zone); } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), - transaction); + AssignedCertificate assigned = new AssignedCertificate(application, instanceName, candidate.get().certificate(), false); + curator.writeAssignedCertificate(assigned, transaction); transaction.commit(); - return certificate; + return assigned; } } } - AssignedCertificate updateLastRequested(AssignedCertificate assignedCertificate) { - AssignedCertificate updated = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); - return updated; + private AssignedCertificate instanceLevelCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, boolean allowPool) { + TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); + Optional<InstanceName> instance = Optional.of(deployment.applicationId().instance()); + Optional<AssignedCertificate> currentCertificate = curator.readAssignedCertificate(application, instance); + final AssignedCertificate assignedCertificate; + if (currentCertificate.isEmpty()) { + Optional<String> generatedId = Optional.empty(); + // Re-use the generated ID contained in an existing certificate (matching this application, this instance, + // or any other instance present in deployment sec), if any. If this exists we provision a new certificate + // containing the same ID + if (!deployment.zoneId().environment().isManuallyDeployed()) { + generatedId = curator.readAssignedCertificates().stream() + .filter(ac -> { + boolean matchingInstance = ac.instance().isPresent() && + deploymentSpec.instance(ac.instance().get()).isPresent(); + return (matchingInstance || ac.instance().isEmpty()) && + ac.application().equals(application); + }) + .map(AssignedCertificate::certificate) + .flatMap(ac -> ac.generatedId().stream()) + .findFirst(); + } + if (allowPool && generatedId.isEmpty()) { + assignedCertificate = assignFromPool(application, instance, deployment.zoneId()); + } else { + if (generatedId.isEmpty()) { + generatedId = Optional.of(generateId()); + } + EndpointCertificate provisionedCertificate = provision(deployment, Optional.empty(), deploymentSpec, generatedId.get()); + // We do not validate the certificate if one has never existed before - because we do not want to + // wait for it to be available before we deploy. This allows the config server to start + // provisioning nodes ASAP, and the risk is small for a new deployment. + assignedCertificate = new AssignedCertificate(application, instance, provisionedCertificate, false); + } + } else { + assignedCertificate = currentCertificate.get().withShouldValidate(!allowPool); + } + return assignedCertificate; } - private Optional<EndpointCertificate> getOrProvision(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { - if (controller.routing().generatedEndpointsEnabled(instance.id())) { - return Optional.of(assignFromPool(instance, zone)); + private AssignedCertificate applicationLevelCertificate(DeploymentId deployment) { + if (deployment.zoneId().environment().isManuallyDeployed()) { + throw new IllegalArgumentException(deployment + " is manually deployed and cannot assign an application-level certificate"); } - Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.id().instance())); - DeploymentId deployment = new DeploymentId(instance.id(), zone); - - if (assignedCertificate.isEmpty()) { - var provisionedCertificate = provisionEndpointCertificate(deployment, Optional.empty(), deploymentSpec); - // We do not verify the certificate if one has never existed before - because we do not want to - // wait for it to be available before we deploy. This allows the config server to start - // provisioning nodes ASAP, and the risk is small for a new deployment. - curator.writeAssignedCertificate(new AssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.id().instance()), provisionedCertificate)); - return Optional.of(provisionedCertificate); - } else { - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); + TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); + Optional<AssignedCertificate> applicationLevelCertificate = curator.readAssignedCertificate(application, Optional.empty()); + if (applicationLevelCertificate.isEmpty()) { + Optional<AssignedCertificate> instanceLevelCertificate = curator.readAssignedCertificate(application, Optional.of(deployment.applicationId().instance())); + // Migrate from instance-level certificate + if (instanceLevelCertificate.isPresent()) { + try (var transaction = new NestedTransaction()) { + AssignedCertificate assignedCertificate = instanceLevelCertificate.get().withoutInstance(); + curator.removeAssignedCertificate(application, Optional.of(deployment.applicationId().instance()), transaction); + curator.writeAssignedCertificate(assignedCertificate, transaction); + transaction.commit(); + return assignedCertificate; + } + } else { + return assignFromPool(application, Optional.empty(), deployment.zoneId()); + } + } + return applicationLevelCertificate.get(); + } + + /** Assign a certificate to given deployment. A new certificate is provisioned (possibly from a pool) and reconfigured as necessary */ + private EndpointCertificate assignTo(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointConfig config) { + // Assign certificate based on endpoint config + AssignedCertificate assignedCertificate = switch (config) { + case legacy, combined -> instanceLevelCertificate(deployment, deploymentSpec, false); + case generated -> deployment.zoneId().environment().isManuallyDeployed() + ? instanceLevelCertificate(deployment, deploymentSpec, true) + : applicationLevelCertificate(deployment); + }; + + // Generate ID if not already present in certificate + Optional<String> generatedId = assignedCertificate.certificate().generatedId(); + if (generatedId.isEmpty()) { + generatedId = Optional.of(generateId()); + assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withGeneratedId(generatedId.get())); + } + + // Ensure all wanted names are present in certificate + List<String> wantedNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, generatedId.get(), config.supportsLegacy()); + Set<String> currentNames = Set.copyOf(assignedCertificate.certificate().requestedDnsSans()); + // TODO(mpolden): Consider requiring exact match for generated as we likely want to remove any legacy names in this case + if (!currentNames.containsAll(wantedNames)) { + EndpointCertificate updatedCertificate = provision(deployment, Optional.of(assignedCertificate.certificate()), deploymentSpec, generatedId.get()); + // Validation is unlikely to succeed in this case, as certificate must be available first. Controller will retry + assignedCertificate = assignedCertificate.with(updatedCertificate) + .withShouldValidate(true); } - // Re-provision certificate if it is missing SANs for the zone we are deploying to - // Skip this validation for now if the cert has a randomized id and should not provision legacy names - Optional<EndpointCertificate> currentCertificate = assignedCertificate.map(AssignedCertificate::certificate); - boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); - - var requiredSansForZone = legacyNames || currentCertificate.get().generatedId().isEmpty() ? - controller.routing().certificateDnsNames(deployment, deploymentSpec) : - List.<String>of(); - - if (!currentCertificate.get().requestedDnsSans().containsAll(requiredSansForZone)) { - var reprovisionedCertificate = - provisionEndpointCertificate(deployment, currentCertificate, deploymentSpec) - .withRootRequestId(currentCertificate.get().rootRequestId()); // We're required to keep the original request ID - curator.writeAssignedCertificate(assignedCertificate.get().with(reprovisionedCertificate)); - // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry - certificateValidator.validate(reprovisionedCertificate, instance.id().serializedForm(), zone, requiredSansForZone); - return Optional.of(reprovisionedCertificate); + // Require that generated ID is always set, for any kind of certificate + if (assignedCertificate.certificate().generatedId().isEmpty()) { + throw new IllegalArgumentException("Certificate for " + deployment + " does not contain generated ID: " + + assignedCertificate.certificate()); } - certificateValidator.validate(currentCertificate.get(), instance.id().serializedForm(), zone, requiredSansForZone); - return currentCertificate; + // Update the time we last requested this certificate. This field is used by EndpointCertificateMaintainer to + // determine stale certificates + assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); + curator.writeAssignedCertificate(assignedCertificate); + + // Validate if we're re-assigned an existing certificate, or if we updated the names of an existing certificate + if (assignedCertificate.shouldValidate()) { + certificateValidator.validate(assignedCertificate.certificate(), deployment.applicationId().serializedForm(), + deployment.zoneId(), wantedNames); + } + + return assignedCertificate.certificate(); } - private EndpointCertificate provisionEndpointCertificate(DeploymentId deployment, - Optional<EndpointCertificate> currentCert, - DeploymentSpec deploymentSpec) { + private String generateId() { + List<String> unassignedIds = curator.readUnassignedCertificates().stream() + .map(UnassignedCertificate::id) + .toList(); + List<String> assignedIds = curator.readAssignedCertificates().stream() + .map(AssignedCertificate::certificate) + .map(EndpointCertificate::generatedId) + .flatMap(Optional::stream) + .toList(); + Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); + String id; + do { + id = GeneratedEndpoint.createPart(controller.random(true)); + } while (allIds.contains(id)); + return id; + } + + private EndpointCertificate provision(DeploymentId deployment, + Optional<EndpointCertificate> current, + DeploymentSpec deploymentSpec, + String generatedId) { List<ZoneId> zonesInSystem = controller.zoneRegistry().zones().controllerUpgraded().ids(); Set<ZoneId> requiredZones = new LinkedHashSet<>(); requiredZones.add(deployment.zoneId()); @@ -214,39 +283,36 @@ public class EndpointCertificates { instanceSpec.get().deploysTo(zone.environment(), zone.region()))) .forEach(requiredZones::add); } - /* TODO(andreer/mpolden): To allow a seamless transition of existing deployments to using generated endpoints, - we need to something like this: - 1) All current certificates must be re-provisioned to contain the same wildcard names - as CertificatePoolMaintainer, and a randomized ID - 2) Generated endpoints must be exposed *before* switching deployment to a - pre-provisioned certificate - 3) Tenants must shift their traffic to generated endpoints - 4) We can switch to the pre-provisioned certificate. This will invalidate - non-generated endpoints - */ - Set<String> requiredNames = requiredZones.stream() + Set<String> wantedNames = requiredZones.stream() .flatMap(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), - deploymentSpec) + deploymentSpec, generatedId, true) .stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); - // Preserve any currently present names that are still valid - List<String> currentNames = currentCert.map(EndpointCertificate::requestedDnsSans) - .orElseGet(List::of); - zonesInSystem.stream() - .map(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), deploymentSpec)) - .filter(currentNames::containsAll) - .forEach(requiredNames::addAll); + // Preserve any currently present names that are still valid (i.e. the name points to a zone found in this system) + Set<String> currentNames = current.map(EndpointCertificate::requestedDnsSans) + .map(Set::copyOf) + .orElseGet(Set::of); + for (var zone : zonesInSystem) { + List<String> wantedNamesZone = controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), + deploymentSpec, + generatedId, + true); + if (currentNames.containsAll(wantedNamesZone)) { + wantedNames.addAll(wantedNamesZone); + } + } - log.log(Level.INFO, String.format("Requesting new endpoint certificate from Cameo for application %s", deployment.applicationId().serializedForm())); - String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); + // Request certificate + LOG.log(Level.INFO, String.format("Requesting new endpoint certificate for application %s", deployment.applicationId().serializedForm())); + String algo = endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); boolean useAlternativeProvider = useAlternateCertProvider.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); String keyPrefix = deployment.applicationId().toFullString(); - var t0 = Instant.now(); - EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(requiredNames), currentCert, algo, useAlternativeProvider); - var t1 = Instant.now(); - log.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); - return endpointCertificate; + Instant t0 = controller.clock().instant(); + EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(wantedNames), current, algo, useAlternativeProvider); + Instant t1 = controller.clock().instant(); + LOG.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); + return endpointCertificate.withGeneratedId(generatedId); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java index db24eb06b48..5e6e495e473 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/CertificatePoolMaintainer.java @@ -2,6 +2,9 @@ package com.yahoo.vespa.hosted.controller.maintenance; import ai.vespa.metrics.ControllerMetrics; +import com.yahoo.config.application.api.DeploymentSpec; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.zone.ZoneId; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.jdisc.Metric; @@ -11,9 +14,9 @@ import com.yahoo.vespa.flags.IntFlag; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; -import com.yahoo.vespa.hosted.controller.application.Endpoint; import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; @@ -30,7 +33,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; /** - * Manages pool of ready-to-use randomized endpoint certificates + * Manages a pool of ready-to-use endpoint certificates. * * @author andreer */ @@ -44,7 +47,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { private final Metric metric; private final Controller controller; private final IntFlag certPoolSize; - private final String dnsSuffix; private final StringFlag endpointCertificateAlgo; private final BooleanFlag useAlternateCertProvider; @@ -58,7 +60,6 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { this.curator = controller.curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); this.metric = metric; - this.dnsSuffix = Endpoint.dnsSuffix(controller.system()); } protected double maintain() { @@ -72,10 +73,10 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { metric.set(ControllerMetrics.CERTIFICATE_POOL_AVAILABLE.baseName(), (poolSize > 0 ? ((double)available/poolSize) : 1.0), metric.createContext(Map.of())); if (certificatePool.size() < poolSize) { - provisionRandomizedCertificate(); + provisionCertificate(); } } catch (Exception e) { - log.log(Level.SEVERE, "Exception caught while maintaining pool of unused randomized endpoint certs", e); + log.log(Level.SEVERE, "Failed to maintain certificate pool", e); return 1.0; } return 0.0; @@ -90,17 +91,17 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { OptionalInt maxCertVersion = secretStore.listSecretVersions(cert.certificate().certName()).stream().mapToInt(i -> i).max(); if (maxKeyVersion.isPresent() && maxCertVersion.equals(maxKeyVersion)) { curator.writeUnassignedCertificate(cert.withState(UnassignedCertificate.State.ready)); - log.log(Level.INFO, "Randomized endpoint cert %s now ready for use".formatted(cert.id())); + log.log(Level.INFO, "Readied certificate %s".formatted(cert.id())); } } catch (SecretNotFoundException s) { // Likely because the certificate is very recently provisioned - ignore till next time - should we log? - log.log(Level.INFO, "Could not yet read secrets for randomized endpoint cert %s - maybe next time ...".formatted(cert.id())); + log.log(Level.INFO, "Cannot ready certificate %s yet, will retry in %s".formatted(cert.id(), interval())); } } } } - private void provisionRandomizedCertificate() { + private void provisionCertificate() { try (Mutex lock = controller.curator().lockCertificatePool()) { Set<String> existingNames = controller.curator().readUnassignedCertificates().stream().map(UnassignedCertificate::id).collect(Collectors.toSet()); @@ -109,27 +110,30 @@ public class CertificatePoolMaintainer extends ControllerMaintainer { .map(EndpointCertificate::generatedId) .forEach(id -> id.ifPresent(existingNames::add)); - String id = generateRandomId(); - while (existingNames.contains(id)) id = generateRandomId(); - - EndpointCertificate f = endpointCertificateProvider.requestCaSignedCertificate( - "preprovisioned.%s".formatted(id), - List.of( - "*.%s.z%s".formatted(id, dnsSuffix), - "*.%s.g%s".formatted(id, dnsSuffix), - "*.%s.a%s".formatted(id, dnsSuffix) - ), - Optional.empty(), - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withGeneratedId(id); - - UnassignedCertificate certificate = new UnassignedCertificate(f, UnassignedCertificate.State.requested); + String id = generateId(); + while (existingNames.contains(id)) id = generateId(); + List<String> dnsNames = wildcardDnsNames(id); + EndpointCertificate cert = endpointCertificateProvider.requestCaSignedCertificate( + "preprovisioned.%s".formatted(id), + dnsNames, + Optional.empty(), + endpointCertificateAlgo.value(), + useAlternateCertProvider.value()).withGeneratedId(id); + + UnassignedCertificate certificate = new UnassignedCertificate(cert, UnassignedCertificate.State.requested); curator.writeUnassignedCertificate(certificate); } } - private String generateRandomId() { + private List<String> wildcardDnsNames(String id) { + DeploymentId defaultDeployment = new DeploymentId(ApplicationId.defaultId(), ZoneId.defaultId()); + return controller.routing().certificateDnsNames(defaultDeployment, // Not used for non-legacy names + DeploymentSpec.empty, // Not used for non-legacy names + id, + false); + } + + private String generateId() { return GeneratedEndpoint.createPart(controller.random(true)); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index 1d4b6e53109..e3e3e347c04 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -3,32 +3,24 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; import com.yahoo.component.annotation.Inject; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; +import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -42,11 +34,9 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -66,9 +56,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at()); - private final StringFlag endpointCertificateAlgo; - private final BooleanFlag useAlternateCertProvider; - private final IntFlag assignRandomizedIdRate; @Inject public EndpointCertificateMaintainer(Controller controller, Duration interval) { @@ -79,9 +66,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); } @Override @@ -92,12 +76,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { updateRefreshedCertificates(); deleteUnusedCertificates(); deleteOrReportUnmanagedCertificates(); - assignRandomizedIds(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); return 1.0; } - return 0.0; } @@ -269,115 +251,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } } - private void assignRandomizedIds() { - List<AssignedCertificate> assignedCertificates = curator.readAssignedCertificates(); - /* - only assign randomized id if: - * instance is present - * randomized id is not already assigned - * feature flag is enabled - */ - assignedCertificates.stream() - .filter(c -> c.instance().isPresent()) - .filter(c -> c.certificate().generatedId().isEmpty()) - .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion - .limit(assignRandomizedIdRate.value()) - .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); - } - - /* - Assign randomized id according to these rules: - * Instance is not mentioned in the deployment spec for this application - -> assume this is a manual deployment. Assign a randomized id to the certificate, save using instance only - * Instance is mentioned in deployment spec: - -> If there is a random endpoint assigned to tenant:application -> use this also for the "instance" certificate - -> Otherwise assign a random endpoint and write to the application and the instance. - */ - private void assignRandomizedId(TenantAndApplicationId tenantAndApplicationId, InstanceName instanceName) { - Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.of(instanceName)); - if (assignedCertificate.isEmpty()) { - log.log(Level.INFO, "Assigned certificate missing for " + tenantAndApplicationId.instance(instanceName).toFullString() + " when assigning randomized id"); - } - // Verify that the assigned certificate still does not have randomized id assigned - if (assignedCertificate.get().certificate().generatedId().isPresent()) return; - - controller().applications().lockApplicationOrThrow(tenantAndApplicationId, application -> { - DeploymentSpec deploymentSpec = application.get().deploymentSpec(); - if (deploymentSpec.instance(instanceName).isPresent()) { - Optional<AssignedCertificate> applicationLevelAssignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.empty()); - assignApplicationRandomId(assignedCertificate.get(), applicationLevelAssignedCertificate); - } else { - assignInstanceRandomId(assignedCertificate.get()); - } - }); - } - - private void assignApplicationRandomId(AssignedCertificate instanceLevelAssignedCertificate, Optional<AssignedCertificate> applicationLevelAssignedCertificate) { - TenantAndApplicationId tenantAndApplicationId = instanceLevelAssignedCertificate.application(); - if (applicationLevelAssignedCertificate.isPresent()) { - // Application level assigned certificate with randomized id already exists. Copy randomized id to instance level certificate and request with random names. - EndpointCertificate withRandomNames = requestRandomNames( - tenantAndApplicationId, - instanceLevelAssignedCertificate.instance(), - applicationLevelAssignedCertificate.get().certificate().generatedId() - .orElseThrow(() -> new IllegalArgumentException("Application certificate already assigned to " + tenantAndApplicationId.toString() + ", but random id is missing")), - Optional.of(instanceLevelAssignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = instanceLevelAssignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } else { - // No application level certificate exists, generate new assigned certificate with the randomized id based names only, then request same names also for instance level cert - String randomId = generateRandomId(); - EndpointCertificate applicationLevelEndpointCert = requestRandomNames(tenantAndApplicationId, Optional.empty(), randomId, Optional.empty()); - AssignedCertificate applicationLevelCert = new AssignedCertificate(tenantAndApplicationId, Optional.empty(), applicationLevelEndpointCert); - - EndpointCertificate instanceLevelEndpointCert = requestRandomNames(tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), randomId, Optional.of(instanceLevelAssignedCertificate.certificate())); - instanceLevelAssignedCertificate = instanceLevelAssignedCertificate.with(instanceLevelEndpointCert); - - // Save both in transaction - try (NestedTransaction transaction = new NestedTransaction()) { - curator.writeAssignedCertificate(instanceLevelAssignedCertificate, transaction); - curator.writeAssignedCertificate(applicationLevelCert, transaction); - transaction.commit(); - } - } - } - - private void assignInstanceRandomId(AssignedCertificate assignedCertificate) { - String randomId = generateRandomId(); - EndpointCertificate withRandomNames = requestRandomNames(assignedCertificate.application(), assignedCertificate.instance(), randomId, Optional.of(assignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = assignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } - - private EndpointCertificate requestRandomNames(TenantAndApplicationId tenantAndApplicationId, Optional<InstanceName> instanceName, String randomId, Optional<EndpointCertificate> previousRequest) { - String dnsSuffix = Endpoint.dnsSuffix(controller().system()); - List<String> newSanDnsEntries = List.of( - "*.%s.z%s".formatted(randomId, dnsSuffix), - "*.%s.g%s".formatted(randomId, dnsSuffix), - "*.%s.a%s".formatted(randomId, dnsSuffix)); - List<String> existingSanDnsEntries = previousRequest.map(EndpointCertificate::requestedDnsSans).orElse(List.of()); - List<String> requestNames = Stream.concat(existingSanDnsEntries.stream(), newSanDnsEntries.stream()).toList(); - String key = instanceName.map(tenantAndApplicationId::instance).map(ApplicationId::toFullString).orElseGet(tenantAndApplicationId::toString); - return endpointCertificateProvider.requestCaSignedCertificate( - key, - requestNames, - previousRequest, - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withGeneratedId(randomId); - } - - private String generateRandomId() { - List<String> unassignedIds = curator.readUnassignedCertificates().stream().map(UnassignedCertificate::id).toList(); - List<String> assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::generatedId).filter(Optional::isPresent).map(Optional::get).toList(); - Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); - String randomId; - do { - randomId = GeneratedEndpoint.createPart(controller().random(true)); - } while (allIds.contains(randomId)); - return randomId; - } - private static String asString(TenantAndApplicationId application, Optional<InstanceName> instanceName) { return application.toString() + instanceName.map(name -> "." + name.value()).orElse(""); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index ea425d59a9f..dceb3921061 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; import java.util.logging.Level; @@ -42,10 +43,16 @@ public class Upgrader extends ControllerMaintainer { private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; + private final Random random; public Upgrader(Controller controller, Duration interval) { + this(controller, interval, controller.random(false)); + } + + Upgrader(Controller controller, Duration interval, Random random) { super(controller, interval); this.curator = controller.curator(); + this.random = random; } /** @@ -75,7 +82,7 @@ public class Upgrader extends ControllerMaintainer { private InstanceList instances(DeploymentStatusList deploymentStatuses) { return InstanceList.from(deploymentStatuses) .withDeclaredJobs() - .shuffle(controller().random(false)) + .shuffle(random) .byIncreasingDeployedVersion() .unpinned(); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index 7923dbb34e3..a2a4cf809b1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; @@ -643,6 +642,10 @@ public class CuratorDb { curator.delete(endpointCertificatePath(application, instanceName)); } + public void removeAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instanceName, NestedTransaction transaction) { + transaction.add(CuratorTransaction.from(CuratorOperations.delete(endpointCertificatePath(application, instanceName).getAbsolute()), curator)); + } + // TODO(mpolden): Remove this. Caller should make an explicit decision to read certificate for a particular instance public Optional<AssignedCertificate> readAssignedCertificate(ApplicationId applicationId) { return readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())); @@ -651,7 +654,7 @@ public class CuratorDb { public Optional<AssignedCertificate> readAssignedCertificate(TenantAndApplicationId application, Optional<InstanceName> instance) { return readSlime(endpointCertificatePath(application, instance)).map(Slime::get) .map(EndpointCertificateSerializer::fromSlime) - .map(cert -> new AssignedCertificate(application, instance, cert)); + .map(cert -> new AssignedCertificate(application, instance, cert, false)); } public List<AssignedCertificate> readAssignedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java index b264f3ea7c5..b38bb73a98a 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java @@ -20,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateSerializer; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import java.util.List; import java.util.Optional; @@ -74,11 +75,11 @@ public class EndpointCertificatesHandler extends ThreadedHttpRequestHandler { public StringResponse reRequestEndpointCertificateFor(String instanceId, boolean ignoreExisting) { ApplicationId applicationId = ApplicationId.fromFullString(instanceId); - if (controller.routing().generatedEndpointsEnabled(applicationId)) { + if (controller.routing().endpointConfig(applicationId) == EndpointConfig.generated) { throw new IllegalArgumentException("Cannot re-request certificate. " + instanceId + " is assigned certificate from a pool"); } try (var lock = curator.lock(TenantAndApplicationId.from(applicationId))) { - AssignedCertificate assignedCertificate = curator.readAssignedCertificate(applicationId) + AssignedCertificate assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())) .orElseThrow(() -> new RestApiException.NotFound("No certificate found for application " + applicationId.serializedForm())); String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java new file mode 100644 index 00000000000..555fd024e47 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +/** + * Endpoint configurations supported for an application. + * + * @author mpolden + */ +public enum EndpointConfig { + + /** Only legacy endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ + legacy, + + /** Legacy and generated endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ + combined, + + /** Only generated endpoints will be published in DNS. Certificate will contain generated names only. Certificate is assigned from a pool */ + generated; + + /** Returns whether this config supports legacy endpoints */ + public boolean supportsLegacy() { + return this == legacy || this == combined; + } + + /** Returns whether this config supports generated endpoints */ + public boolean supportsGenerated() { + return this == combined || this == generated; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java index 78a2be3bc5b..63b17a087f2 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java @@ -28,13 +28,13 @@ import java.util.stream.Collectors; public record PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, - Optional<EndpointCertificate> certificate) { + EndpointCertificate certificate) { - public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, Optional<EndpointCertificate> certificate) { + public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List<AssignedRotation> rotations, EndpointCertificate certificate) { this.deployment = Objects.requireNonNull(deployment); this.endpoints = Objects.requireNonNull(endpoints); this.rotations = List.copyOf(Objects.requireNonNull(rotations)); - this.certificate = Objects.requireNonNull(certificate); + this.certificate = requireMatchingSans(certificate, endpoints); } /** Returns the endpoints contained in this as {@link com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint} */ @@ -101,4 +101,15 @@ public record PreparedEndpoints(DeploymentId deployment, }; } + private static EndpointCertificate requireMatchingSans(EndpointCertificate certificate, EndpointList endpoints) { + Objects.requireNonNull(certificate); + for (var endpoint : endpoints.not().scope(Endpoint.Scope.weighted)) { // Weighted endpoints are not present in certificate + if (!certificate.sanMatches(endpoint.dnsName())) { + throw new IllegalArgumentException(endpoint + " has no matching SAN. Certificate contains " + + certificate.requestedDnsSans()); + } + } + return certificate; + } + } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index 2cad499899c..50e65187835 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -45,7 +45,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { * * @return the container endpoints relevant for this deployment, as declared in deployment spec */ - public final PreparedEndpoints prepare(BasicServicesXml services, Optional<EndpointCertificate> certificate, LockedApplication application) { + public final PreparedEndpoints prepare(BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { return routing.prepare(deployment, services, certificate, application); } |