From ea5847e39a0147d5c7de5f579bee8cf4d3441968 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Thu, 5 Oct 2023 13:00:43 +0200 Subject: Support building wildcard names for generated endpoints --- .../hosted/controller/application/EndpointTest.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) (limited to 'controller-server/src/test/java') diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java index fbc5567101f..8667cd335c8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/EndpointTest.java @@ -370,6 +370,24 @@ public class EndpointTest { "dead2bad.deadbeef.a.vespa-app.cloud", Endpoint.of(TenantAndApplicationId.from(instance1)).targetApplication(EndpointId.of("foo"), deployment) .generatedFrom(ge2) + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Wildcard endpoint for zone + "*.deadbeef.z.vespa-app.cloud", + Endpoint.of(instance1) + .wildcardGenerated(ge1.applicationPart(), Endpoint.Scope.zone) + .certificateName() + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Wildcard endpoint for global + "*.deadbeef.g.vespa-app.cloud", + Endpoint.of(instance1) + .wildcardGenerated(ge1.applicationPart(), Endpoint.Scope.global) + .certificateName() + .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public), + // Wildcard endpoint for application + "*.deadbeef.a.vespa-app.cloud", + Endpoint.of(instance1) + .wildcardGenerated(ge1.applicationPart(), Endpoint.Scope.application) + .certificateName() .routingMethod(RoutingMethod.exclusive).on(Port.tls()).in(SystemName.Public) ); tests.forEach((expected, endpoint) -> assertEquals(expected, endpoint.dnsName())); -- cgit v1.2.3 From 1a776a01494f1a0291169962814361be3ffca714 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 6 Oct 2023 10:38:55 +0200 Subject: Fix Random seeding in UpgraderTest --- .../vespa/hosted/controller/maintenance/Upgrader.java | 9 ++++++++- .../yahoo/vespa/hosted/controller/ControllerTester.java | 14 ++------------ .../vespa/hosted/controller/maintenance/UpgraderTest.java | 8 ++++---- 3 files changed, 14 insertions(+), 17 deletions(-) (limited to 'controller-server/src/test/java') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java index a929a1d7af8..7dab7206832 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/Upgrader.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Random; import java.util.Set; import java.util.function.UnaryOperator; import java.util.logging.Level; @@ -42,10 +43,16 @@ public class Upgrader extends ControllerMaintainer { private static final Logger log = Logger.getLogger(Upgrader.class.getName()); private final CuratorDb curator; + private final Random random; public Upgrader(Controller controller, Duration interval) { + this(controller, interval, controller.random(false)); + } + + Upgrader(Controller controller, Duration interval, Random random) { super(controller, interval); this.curator = controller.curator(); + this.random = random; } /** @@ -75,7 +82,7 @@ public class Upgrader extends ControllerMaintainer { private InstanceList instances(DeploymentStatusList deploymentStatuses) { return InstanceList.from(deploymentStatuses) .withDeclaredJobs() - .shuffle(controller().random(false)) + .shuffle(random) .byIncreasingDeployedVersion() .unpinned(); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java index d9b95a53a0e..a4c20b1ddb5 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTester.java @@ -3,9 +3,7 @@ package com.yahoo.vespa.hosted.controller; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.Environment; import com.yahoo.config.provision.HostName; -import com.yahoo.config.provision.RegionName; import com.yahoo.config.provision.SystemName; import com.yahoo.config.provision.TenantName; import com.yahoo.config.provision.zone.RoutingMethod; @@ -20,6 +18,7 @@ import com.yahoo.vespa.athenz.api.OAuthCredentials; import com.yahoo.vespa.flags.FlagSource; import com.yahoo.vespa.flags.InMemoryFlagSource; import com.yahoo.vespa.flags.PermanentFlags; +import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.api.identifiers.Property; import com.yahoo.vespa.hosted.controller.api.identifiers.PropertyId; import com.yahoo.vespa.hosted.controller.api.integration.athenz.AthenzClientFactoryMock; @@ -54,7 +53,6 @@ import com.yahoo.vespa.hosted.controller.security.Credentials; import com.yahoo.vespa.hosted.controller.security.TenantSpec; import com.yahoo.vespa.hosted.controller.tenant.AthenzTenant; import com.yahoo.vespa.hosted.controller.tenant.Tenant; -import com.yahoo.vespa.hosted.controller.api.identifiers.ControllerVersion; import com.yahoo.vespa.hosted.controller.versions.VersionStatus; import com.yahoo.vespa.hosted.rotation.config.RotationsConfig; import com.yahoo.yolean.concurrent.Sleeper; @@ -289,14 +287,6 @@ public final class ControllerTester { return controller().clock().instant().atOffset(ZoneOffset.UTC).getHour(); } - public ZoneId toZone(Environment environment) { - return switch (environment) { - case dev, test -> ZoneId.from(environment, RegionName.from("us-east-1")); - case staging -> ZoneId.from(environment, RegionName.from("us-east-3")); - default -> ZoneId.from(environment, RegionName.from("us-west-1")); - }; - } - public AthenzDomain createDomainWithAdmin(String domainName, AthenzUser user) { AthenzDomain domain = new AthenzDomain(domainName); athenzDb.getOrCreateDomain(domain).admin(user); @@ -405,7 +395,7 @@ public final class ControllerTester { RotationsConfig.Builder builder = new RotationsConfig.Builder(); for (int i = 1; i <= availableRotations; i++) { String id = Text.format("%02d", i); - builder = builder.rotations("rotation-id-" + id, "rotation-fqdn-" + id); + builder.rotations("rotation-id-" + id, "rotation-fqdn-" + id); } return new RotationsConfig(builder); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java index f1e8697cf41..560ac73f320 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/UpgraderTest.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.yahoo.component.Version; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.zone.ZoneId; -import com.yahoo.test.ManualClock; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RevisionId; import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId; import com.yahoo.vespa.hosted.controller.application.Change; @@ -26,6 +25,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Random; import java.util.Set; import java.util.stream.Collectors; @@ -40,7 +40,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.Cha import static com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel.PLATFORM; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; /** @@ -1100,8 +1099,9 @@ public class UpgraderTest { default2.instanceId(), default2); // Throttle upgrades per run - ((ManualClock) tester.controller().clock()).setInstant(Instant.ofEpochMilli(1589787107000L)); // Fixed random seed - Upgrader upgrader = new Upgrader(tester.controller(), Duration.ofMinutes(10)); + Upgrader upgrader = new Upgrader(tester.controller(), + Duration.ofMinutes(10), + new Random(1589787107000L)); // Fixed random seed upgrader.setUpgradesPerMinute(0.1); // Trigger some upgrades -- cgit v1.2.3 From 9eb903f4d1d571be513a70c59e49d38d7a986220 Mon Sep 17 00:00:00 2001 From: Martin Polden Date: Fri, 6 Oct 2023 14:57:17 +0200 Subject: Refactor certificate assignment and migration --- .../hosted/controller/ApplicationController.java | 13 +- .../vespa/hosted/controller/RoutingController.java | 54 +++- .../certificate/AssignedCertificate.java | 13 +- .../certificate/EndpointCertificates.java | 332 ++++++++++++--------- .../maintenance/EndpointCertificateMaintainer.java | 129 +------- .../hosted/controller/persistence/CuratorDb.java | 7 +- .../certificate/EndpointCertificatesHandler.java | 5 +- .../hosted/controller/routing/EndpointConfig.java | 30 ++ .../controller/routing/PreparedEndpoints.java | 11 +- .../routing/context/DeploymentRoutingContext.java | 2 +- .../vespa/hosted/controller/ControllerTest.java | 30 +- .../certificate/EndpointCertificatesTest.java | 331 +++++++++++++------- .../EndpointCertificateMaintainerTest.java | 77 +---- .../controller/routing/RoutingPoliciesTest.java | 108 +++---- .../src/main/java/com/yahoo/vespa/flags/Flags.java | 7 + 15 files changed, 602 insertions(+), 547 deletions(-) create mode 100644 controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java (limited to 'controller-server/src/test/java') diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java index 5e4d73954ae..5abd2deda54 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java @@ -524,7 +524,7 @@ public class ApplicationController { try (Mutex lock = lock(applicationId)) { LockedApplication application = new LockedApplication(requireApplication(applicationId), lock); application.get().revisions().last().map(ApplicationVersion::id).ifPresent(lastRevision::set); - return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger); + return prepareEndpoints(deployment, job, application, applicationPackage, deployLogger, lock); } }; @@ -569,13 +569,16 @@ public class ApplicationController { private PreparedEndpoints prepareEndpoints(DeploymentId deployment, JobId job, LockedApplication application, ApplicationPackageStream applicationPackage, - Consumer deployLogger) { + Consumer deployLogger, + Mutex applicationLock) { Instance instance = application.get().require(job.application().instance()); Tags tags = applicationPackage.truncatedPackage().deploymentSpec().instance(instance.name()) .map(DeploymentInstanceSpec::tags) .orElseGet(Tags::empty); - Optional certificate = endpointCertificates.get(instance, deployment.zoneId(), applicationPackage.truncatedPackage().deploymentSpec()); - certificate.ifPresent(e -> deployLogger.accept("Using CA signed certificate version %s".formatted(e.version()))); + EndpointCertificate certificate = endpointCertificates.get(deployment, + applicationPackage.truncatedPackage().deploymentSpec(), + applicationLock); + deployLogger.accept("Using CA signed certificate version %s".formatted(certificate.version())); BasicServicesXml services = applicationPackage.truncatedPackage().services(deployment, tags); return controller.routing().of(deployment).prepare(services, certificate, application); } @@ -696,7 +699,7 @@ public class ApplicationController { if (preparedEndpoints == null) return DeploymentEndpoints.none; PreparedEndpoints prepared = preparedEndpoints.get(); generatedEndpoints.set(prepared.endpoints().generated()); - return new DeploymentEndpoints(prepared.containerEndpoints(), prepared.certificate()); + return new DeploymentEndpoints(prepared.containerEndpoints(), Optional.of(prepared.certificate())); }; Supplier> dataplaneTokenVersions = () -> { Tags tags = applicationPackage.truncatedPackage().deploymentSpec() diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java index b763af1af9d..74a96b9b5e3 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java @@ -32,6 +32,7 @@ import com.yahoo.vespa.hosted.controller.application.SystemApplication; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.BasicServicesXml; import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import com.yahoo.vespa.hosted.controller.routing.GeneratedEndpointList; import com.yahoo.vespa.hosted.controller.routing.PreparedEndpoints; import com.yahoo.vespa.hosted.controller.routing.RoutingId; @@ -121,8 +122,21 @@ public class RoutingController { return rotationRepository; } + /** Returns the endpoint config to use for given instance */ + public EndpointConfig endpointConfig(ApplicationId instance) { + // TODO(mpolden): Switch to reading endpoint-config flag + if (legacyEndpointsEnabled(instance)) { + if (generatedEndpointsEnabled(instance)) { + return EndpointConfig.combined; + } else { + return EndpointConfig.legacy; + } + } + return EndpointConfig.generated; + } + /** Prepares and returns the endpoints relevant for given deployment */ - public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, Optional certificate, LockedApplication application) { + public PreparedEndpoints prepare(DeploymentId deployment, BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { EndpointList endpoints = EndpointList.EMPTY; DeploymentSpec spec = application.get().deploymentSpec(); @@ -136,7 +150,7 @@ public class RoutingController { // Add zone-scoped endpoints Map> generatedForDeclaredEndpoints = new HashMap<>(); Set clustersWithToken = new HashSet<>(); - boolean generatedEndpointsEnabled = generatedEndpointsEnabled(deployment.applicationId()); + EndpointConfig config = endpointConfig(deployment.applicationId()); RoutingPolicyList applicationPolicies = policies().read(TenantAndApplicationId.from(deployment.applicationId())); RoutingPolicyList deploymentPolicies = applicationPolicies.deployment(deployment); for (var container : services.containers()) { @@ -153,7 +167,7 @@ public class RoutingController { if (tokenSupported) { generatedForCluster = generateEndpoints(AuthMethod.token, certificate, Optional.empty(), generatedForCluster); } - GeneratedEndpointList generatedEndpoints = generatedEndpointsEnabled ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; + GeneratedEndpointList generatedEndpoints = config.supportsGenerated() ? GeneratedEndpointList.copyOf(generatedForCluster) : GeneratedEndpointList.EMPTY; endpoints = endpoints.and(endpointsOf(deployment, clusterId, generatedEndpoints).scope(Scope.zone)); } @@ -185,7 +199,7 @@ public class RoutingController { return generatedEndpoints; }); }); - Map generatedEndpoints = generatedEndpointsEnabled + Map generatedEndpoints = config.supportsGenerated() ? generatedForDeclaredEndpoints.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, kv -> GeneratedEndpointList.copyOf(kv.getValue()))) @@ -380,7 +394,24 @@ public class RoutingController { } /** Returns certificate DNS names (CN and SAN values) for given deployment */ - public List certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { + public List certificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec, String generatedId, boolean legacy) { + List endpointDnsNames = new ArrayList<>(); + if (legacy) { + endpointDnsNames.addAll(legacyCertificateDnsNames(deployment, deploymentSpec)); + } + for (Scope scope : List.of(Scope.zone, Scope.global, Scope.application)) { + endpointDnsNames.add(Endpoint.of(deployment.applicationId()) + .wildcardGenerated(generatedId, scope) + .routingMethod(RoutingMethod.exclusive) + .on(Port.tls()) + .certificateName() + .in(controller.system()) + .dnsName()); + } + return Collections.unmodifiableList(endpointDnsNames); + } + + private List legacyCertificateDnsNames(DeploymentId deployment, DeploymentSpec deploymentSpec) { List endpointDnsNames = new ArrayList<>(); // We add first an endpoint name based on a hash of the application ID, @@ -447,10 +478,7 @@ public class RoutingController { } private EndpointList filterEndpoints(ApplicationId instance, EndpointList endpoints) { - if (generatedEndpointsEnabled(instance) && !legacyEndpointsEnabled(instance)) { - return endpoints.generated(); - } - return endpoints; + return endpointConfig(instance) == EndpointConfig.generated ? endpoints.generated() : endpoints; } private void registerRotationEndpointsInDns(PreparedEndpoints prepared) { @@ -491,13 +519,13 @@ public class RoutingController { } /** Returns generated endpoints. A new endpoint is generated if no matching endpoint already exists */ - private List generateEndpoints(AuthMethod authMethod, Optional certificate, + private List generateEndpoints(AuthMethod authMethod, EndpointCertificate certificate, Optional declaredEndpoint, List current) { if (current.stream().anyMatch(e -> e.authMethod() == authMethod && e.endpoint().equals(declaredEndpoint))) { return current; } - Optional applicationPart = certificate.flatMap(EndpointCertificate::generatedId); + Optional applicationPart = certificate.generatedId(); if (applicationPart.isPresent()) { current = new ArrayList<>(current); current.add(new GeneratedEndpoint(GeneratedEndpoint.createPart(controller.random(true)), @@ -572,14 +600,14 @@ public class RoutingController { return Collections.unmodifiableList(routingMethods); } - public boolean generatedEndpointsEnabled(ApplicationId instance) { + private boolean generatedEndpointsEnabled(ApplicationId instance) { return generatedEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) .value(); } - public boolean legacyEndpointsEnabled(ApplicationId instance) { + private boolean legacyEndpointsEnabled(ApplicationId instance) { return legacyEndpoints.with(FetchVector.Dimension.INSTANCE_ID, instance.serializedForm()) .with(FetchVector.Dimension.TENANT_ID, instance.tenant().value()) .with(FetchVector.Dimension.APPLICATION_ID, TenantAndApplicationId.from(instance).serialized()) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java index 7d3bcf8bdaa..d52b01b4446 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/AssignedCertificate.java @@ -15,10 +15,19 @@ import java.util.Optional; */ public record AssignedCertificate(TenantAndApplicationId application, Optional instance, - EndpointCertificate certificate) { + EndpointCertificate certificate, + boolean shouldValidate) { public AssignedCertificate with(EndpointCertificate certificate) { - return new AssignedCertificate(application, instance, certificate); + return new AssignedCertificate(application, instance, certificate, shouldValidate); + } + + public AssignedCertificate withoutInstance() { + return new AssignedCertificate(application, Optional.empty(), certificate, shouldValidate); + } + + public AssignedCertificate withShouldValidate(boolean shouldValidate) { + return new AssignedCertificate(application, instance, certificate, shouldValidate); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java index ec2ef4b7ff8..255adcb521c 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificates.java @@ -11,18 +11,18 @@ import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; -import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.flags.PermanentFlags; import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Controller; -import com.yahoo.vespa.hosted.controller.Instance; import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidator; import com.yahoo.vespa.hosted.controller.api.integration.secrets.GcpSecretStore; +import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import java.time.Clock; import java.time.Duration; @@ -30,26 +30,29 @@ import java.time.Instant; import java.util.Comparator; import java.util.LinkedHashSet; import java.util.List; +import java.util.Objects; import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate.State; /** - * Looks up stored endpoint certificate, provisions new certificates if none is found, - * and re-provisions the certificate if the deploying-to zone is not covered. + * This provisions, assigns and updates the certificate for a given deployment. * * See also {@link com.yahoo.vespa.hosted.controller.maintenance.EndpointCertificateMaintainer}, which handles * refreshes, deletions and triggers deployments. * * @author andreer + * @author mpolden */ public class EndpointCertificates { - private static final Logger log = Logger.getLogger(EndpointCertificates.class.getName()); + private static final Logger LOG = Logger.getLogger(EndpointCertificates.class.getName()); + private static final Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time private final Controller controller; private final CuratorDb curator; @@ -58,150 +61,216 @@ public class EndpointCertificates { private final EndpointCertificateValidator certificateValidator; private final BooleanFlag useAlternateCertProvider; private final StringFlag endpointCertificateAlgo; - private final BooleanFlag assignLegacyNames; - private final static Duration GCP_CERTIFICATE_EXPIRY_TIME = Duration.ofDays(100); // 100 days, 10 more than notAfter time public EndpointCertificates(Controller controller, EndpointCertificateProvider certificateProvider, EndpointCertificateValidator certificateValidator) { this.controller = controller; this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignLegacyNames = Flags.LEGACY_ENDPOINTS.bindTo(controller.flagSource()); this.curator = controller.curator(); this.clock = controller.clock(); this.certificateProvider = certificateProvider; this.certificateValidator = certificateValidator; } - /** Returns a suitable certificate for endpoints of given instance and zone */ - public Optional get(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { + /** Returns a suitable certificate for endpoints of given deployment */ + public EndpointCertificate get(DeploymentId deployment, DeploymentSpec deploymentSpec, Mutex applicationLock) { + Objects.requireNonNull(applicationLock); Instant start = clock.instant(); - Optional cert = getOrProvision(instance, zone, deploymentSpec); + EndpointConfig config = controller.routing().endpointConfig(deployment.applicationId()); + EndpointCertificate certificate = assignTo(deployment, deploymentSpec, config); Duration duration = Duration.between(start, clock.instant()); - if (duration.toSeconds() > 30) - log.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds())); - - if (controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(zone)) { // Until CKMS is available from GCP - if (cert.isPresent()) { - // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time - certificateValidator.validate(cert.get(), instance.id().serializedForm(), zone, controller.routing().certificateDnsNames(new DeploymentId(instance.id(), zone), deploymentSpec)); - GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); - String mangledCertName = "endpointCert_" + cert.get().certName().replace('.', '_') + "-v" + cert.get().version(); // Google cloud does not accept dots in secrets, but they accept underscores - String mangledKeyName = "endpointCert_" + cert.get().keyName().replace('.', '_') + "-v" + cert.get().version(); // Google cloud does not accept dots in secrets, but they accept underscores - if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { - gcpSecretStore.setSecret(mangledCertName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledCertName, - controller.secretStore().getSecret(cert.get().certName(), cert.get().version())); - } - if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { - gcpSecretStore.setSecret(mangledKeyName, - Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), - "endpoint-cert-accessor"); - gcpSecretStore.addSecretVersion(mangledKeyName, - controller.secretStore().getSecret(cert.get().keyName(), cert.get().version())); - } - - return Optional.of(cert.get().withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName)); - } + if (duration.toSeconds() > 30) { + LOG.log(Level.INFO, Text.format("Getting endpoint certificate for %s took %d seconds!", deployment.applicationId().serializedForm(), duration.toSeconds())); } + if (isGcp(deployment)) { + // This is needed until CKMS is available from GCP + return validateGcpCertificate(deployment, deploymentSpec, certificate, config); + } + return certificate; + } - return cert; + private boolean isGcp(DeploymentId deployment) { + return controller.zoneRegistry().zones().all().in(CloudName.GCP).ids().contains(deployment.zoneId()); } - private EndpointCertificate assignFromPool(Instance instance, ZoneId zone) { - // For deployments to manually deployed environments: use per instance certificate - // For all other environments (apply in order): - // * Use per instance certificate if it exists and is assigned a randomized id - // * Use per application certificate if it exits and is assigned a randomized id - // * Assign from pool - - TenantAndApplicationId application = TenantAndApplicationId.from(instance.id()); - Optional perInstanceAssignedCertificate = curator.readAssignedCertificate(application, Optional.of(instance.name())); - if (perInstanceAssignedCertificate.isPresent() && perInstanceAssignedCertificate.get().certificate().generatedId().isPresent()) { - return updateLastRequested(perInstanceAssignedCertificate.get()).certificate(); - } else if (! zone.environment().isManuallyDeployed()) { - Optional perApplicationAssignedCertificate = curator.readAssignedCertificate(application, Optional.empty()); - if (perApplicationAssignedCertificate.isPresent() && perApplicationAssignedCertificate.get().certificate().generatedId().isPresent()) { - return updateLastRequested(perApplicationAssignedCertificate.get()).certificate(); - } + private EndpointCertificate validateGcpCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointCertificate certificate, EndpointConfig config) { + // Validate before copying cert to GCP. This will ensure we don't bug out on the first deployment, but will take more time + List dnsNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, certificate.generatedId().get(), config.supportsLegacy()); + certificateValidator.validate(certificate, deployment.applicationId().serializedForm(), deployment.zoneId(), dnsNames); + GcpSecretStore gcpSecretStore = controller.serviceRegistry().gcpSecretStore(); + String mangledCertName = "endpointCert_" + certificate.certName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores + String mangledKeyName = "endpointCert_" + certificate.keyName().replace('.', '_') + "-v" + certificate.version(); // Google cloud does not accept dots in secrets, but they accept underscores + if (gcpSecretStore.getLatestSecretVersion(mangledCertName) == null) { + gcpSecretStore.setSecret(mangledCertName, + Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), + "endpoint-cert-accessor"); + gcpSecretStore.addSecretVersion(mangledCertName, + controller.secretStore().getSecret(certificate.certName(), certificate.version())); + } + if (gcpSecretStore.getLatestSecretVersion(mangledKeyName) == null) { + gcpSecretStore.setSecret(mangledKeyName, + Optional.of(GCP_CERTIFICATE_EXPIRY_TIME), + "endpoint-cert-accessor"); + gcpSecretStore.addSecretVersion(mangledKeyName, + controller.secretStore().getSecret(certificate.keyName(), certificate.version())); } + return certificate.withVersion(1).withKeyName(mangledKeyName).withCertName(mangledCertName); + } - // For new applications which is assigned from pool we follow these rules: - // Assign certificate per instance only in manually deployed environments. In other environments, we share the - // certificate because application endpoints can span instances - Optional instanceName = zone.environment().isManuallyDeployed() ? Optional.of(instance.name()) : Optional.empty(); + private AssignedCertificate assignFromPool(TenantAndApplicationId application, Optional instanceName, ZoneId zone) { try (Mutex lock = controller.curator().lockCertificatePool()) { Optional candidate = curator.readUnassignedCertificates().stream() .filter(pc -> pc.state() == State.ready) .min(Comparator.comparingLong(pc -> pc.certificate().lastRequested())); if (candidate.isEmpty()) { - throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + instance.id() + " in " + zone); + throw new IllegalArgumentException("No endpoint certificate available in pool, for deployment of " + + application + instanceName.map(i -> "." + i.value()).orElse("") + + " in " + zone); } try (NestedTransaction transaction = new NestedTransaction()) { curator.removeUnassignedCertificate(candidate.get(), transaction); - EndpointCertificate certificate = candidate.get().certificate().withLastRequested(clock.instant().getEpochSecond()); - curator.writeAssignedCertificate(new AssignedCertificate(application, instanceName, certificate), - transaction); + AssignedCertificate assigned = new AssignedCertificate(application, instanceName, candidate.get().certificate(), false); + curator.writeAssignedCertificate(assigned, transaction); transaction.commit(); - return certificate; + return assigned; } } } - AssignedCertificate updateLastRequested(AssignedCertificate assignedCertificate) { - AssignedCertificate updated = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); - return updated; + private AssignedCertificate instanceLevelCertificate(DeploymentId deployment, DeploymentSpec deploymentSpec, boolean allowPool) { + TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); + Optional instance = Optional.of(deployment.applicationId().instance()); + Optional currentCertificate = curator.readAssignedCertificate(application, instance); + final AssignedCertificate assignedCertificate; + if (currentCertificate.isEmpty()) { + Optional generatedId = Optional.empty(); + // Re-use the generated ID contained in an existing certificate (matching this application, this instance, + // or any other instance present in deployment sec), if any. If this exists we provision a new certificate + // containing the same ID + if (!deployment.zoneId().environment().isManuallyDeployed()) { + generatedId = curator.readAssignedCertificates().stream() + .filter(ac -> { + boolean matchingInstance = ac.instance().isPresent() && + deploymentSpec.instance(ac.instance().get()).isPresent(); + return (matchingInstance || ac.instance().isEmpty()) && + ac.application().equals(application); + }) + .map(AssignedCertificate::certificate) + .flatMap(ac -> ac.generatedId().stream()) + .findFirst(); + } + if (allowPool && generatedId.isEmpty()) { + assignedCertificate = assignFromPool(application, instance, deployment.zoneId()); + } else { + if (generatedId.isEmpty()) { + generatedId = Optional.of(generateId()); + } + EndpointCertificate provisionedCertificate = provision(deployment, Optional.empty(), deploymentSpec, generatedId.get()); + // We do not validate the certificate if one has never existed before - because we do not want to + // wait for it to be available before we deploy. This allows the config server to start + // provisioning nodes ASAP, and the risk is small for a new deployment. + assignedCertificate = new AssignedCertificate(application, instance, provisionedCertificate, false); + } + } else { + assignedCertificate = currentCertificate.get().withShouldValidate(!allowPool); + } + return assignedCertificate; } - private Optional getOrProvision(Instance instance, ZoneId zone, DeploymentSpec deploymentSpec) { - if (controller.routing().generatedEndpointsEnabled(instance.id())) { - return Optional.of(assignFromPool(instance, zone)); + private AssignedCertificate applicationLevelCertificate(DeploymentId deployment) { + if (deployment.zoneId().environment().isManuallyDeployed()) { + throw new IllegalArgumentException(deployment + " is manually deployed and cannot assign an application-level certificate"); } - Optional assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.id().instance())); - DeploymentId deployment = new DeploymentId(instance.id(), zone); - - if (assignedCertificate.isEmpty()) { - var provisionedCertificate = provisionEndpointCertificate(deployment, Optional.empty(), deploymentSpec); - // We do not verify the certificate if one has never existed before - because we do not want to - // wait for it to be available before we deploy. This allows the config server to start - // provisioning nodes ASAP, and the risk is small for a new deployment. - curator.writeAssignedCertificate(new AssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.of(instance.id().instance()), provisionedCertificate)); - return Optional.of(provisionedCertificate); - } else { - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withLastRequested(clock.instant().getEpochSecond())); - curator.writeAssignedCertificate(updated); + TenantAndApplicationId application = TenantAndApplicationId.from(deployment.applicationId()); + Optional applicationLevelCertificate = curator.readAssignedCertificate(application, Optional.empty()); + if (applicationLevelCertificate.isEmpty()) { + Optional instanceLevelCertificate = curator.readAssignedCertificate(application, Optional.of(deployment.applicationId().instance())); + // Migrate from instance-level certificate + if (instanceLevelCertificate.isPresent()) { + try (var transaction = new NestedTransaction()) { + AssignedCertificate assignedCertificate = instanceLevelCertificate.get().withoutInstance(); + curator.removeAssignedCertificate(application, Optional.of(deployment.applicationId().instance()), transaction); + curator.writeAssignedCertificate(assignedCertificate, transaction); + transaction.commit(); + return assignedCertificate; + } + } else { + return assignFromPool(application, Optional.empty(), deployment.zoneId()); + } + } + return applicationLevelCertificate.get(); + } + + /** Assign a certificate to given deployment. A new certificate is provisioned (possibly from a pool) and reconfigured as necessary */ + private EndpointCertificate assignTo(DeploymentId deployment, DeploymentSpec deploymentSpec, EndpointConfig config) { + // Assign certificate based on endpoint config + AssignedCertificate assignedCertificate = switch (config) { + case legacy, combined -> instanceLevelCertificate(deployment, deploymentSpec, false); + case generated -> deployment.zoneId().environment().isManuallyDeployed() + ? instanceLevelCertificate(deployment, deploymentSpec, true) + : applicationLevelCertificate(deployment); + }; + + // Generate ID if not already present in certificate + Optional generatedId = assignedCertificate.certificate().generatedId(); + if (generatedId.isEmpty()) { + generatedId = Optional.of(generateId()); + assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withGeneratedId(generatedId.get())); + } + + // Ensure all wanted names are present in certificate + List wantedNames = controller.routing().certificateDnsNames(deployment, deploymentSpec, generatedId.get(), config.supportsLegacy()); + Set currentNames = Set.copyOf(assignedCertificate.certificate().requestedDnsSans()); + // TODO(mpolden): Consider requiring exact match for generated as we likely want to remove any legacy names in this case + if (!currentNames.containsAll(wantedNames)) { + EndpointCertificate updatedCertificate = provision(deployment, Optional.of(assignedCertificate.certificate()), deploymentSpec, generatedId.get()); + // Validation is unlikely to succeed in this case, as certificate must be available first. Controller will retry + assignedCertificate = assignedCertificate.with(updatedCertificate) + .withShouldValidate(true); } - // Re-provision certificate if it is missing SANs for the zone we are deploying to - // Skip this validation for now if the cert has a randomized id and should not provision legacy names - Optional currentCertificate = assignedCertificate.map(AssignedCertificate::certificate); - boolean legacyNames = assignLegacyNames.with(FetchVector.Dimension.INSTANCE_ID, instance.id().serializedForm()) - .with(FetchVector.Dimension.APPLICATION_ID, instance.id().toSerializedFormWithoutInstance()).value(); - - var requiredSansForZone = legacyNames || currentCertificate.get().generatedId().isEmpty() ? - controller.routing().certificateDnsNames(deployment, deploymentSpec) : - List.of(); - - if (!currentCertificate.get().requestedDnsSans().containsAll(requiredSansForZone)) { - var reprovisionedCertificate = - provisionEndpointCertificate(deployment, currentCertificate, deploymentSpec) - .withRootRequestId(currentCertificate.get().rootRequestId()); // We're required to keep the original request ID - curator.writeAssignedCertificate(assignedCertificate.get().with(reprovisionedCertificate)); - // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry - certificateValidator.validate(reprovisionedCertificate, instance.id().serializedForm(), zone, requiredSansForZone); - return Optional.of(reprovisionedCertificate); + // Require that generated ID is always set, for any kind of certificate + if (assignedCertificate.certificate().generatedId().isEmpty()) { + throw new IllegalArgumentException("Certificate for " + deployment + " does not contain generated ID: " + + assignedCertificate.certificate()); } - certificateValidator.validate(currentCertificate.get(), instance.id().serializedForm(), zone, requiredSansForZone); - return currentCertificate; + // Update the time we last requested this certificate. This field is used by EndpointCertificateMaintainer to + // determine stale certificates + assignedCertificate = assignedCertificate.with(assignedCertificate.certificate().withLastRequested(clock.instant().getEpochSecond())); + curator.writeAssignedCertificate(assignedCertificate); + + // Validate if we're re-assigned an existing certificate, or if we updated the names of an existing certificate + if (assignedCertificate.shouldValidate()) { + certificateValidator.validate(assignedCertificate.certificate(), deployment.applicationId().serializedForm(), + deployment.zoneId(), wantedNames); + } + + return assignedCertificate.certificate(); } - private EndpointCertificate provisionEndpointCertificate(DeploymentId deployment, - Optional currentCert, - DeploymentSpec deploymentSpec) { + private String generateId() { + List unassignedIds = curator.readUnassignedCertificates().stream() + .map(UnassignedCertificate::id) + .toList(); + List assignedIds = curator.readAssignedCertificates().stream() + .map(AssignedCertificate::certificate) + .map(EndpointCertificate::generatedId) + .flatMap(Optional::stream) + .toList(); + Set allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); + String id; + do { + id = GeneratedEndpoint.createPart(controller.random(true)); + } while (allIds.contains(id)); + return id; + } + + private EndpointCertificate provision(DeploymentId deployment, + Optional current, + DeploymentSpec deploymentSpec, + String generatedId) { List zonesInSystem = controller.zoneRegistry().zones().controllerUpgraded().ids(); Set requiredZones = new LinkedHashSet<>(); requiredZones.add(deployment.zoneId()); @@ -214,39 +283,36 @@ public class EndpointCertificates { instanceSpec.get().deploysTo(zone.environment(), zone.region()))) .forEach(requiredZones::add); } - /* TODO(andreer/mpolden): To allow a seamless transition of existing deployments to using generated endpoints, - we need to something like this: - 1) All current certificates must be re-provisioned to contain the same wildcard names - as CertificatePoolMaintainer, and a randomized ID - 2) Generated endpoints must be exposed *before* switching deployment to a - pre-provisioned certificate - 3) Tenants must shift their traffic to generated endpoints - 4) We can switch to the pre-provisioned certificate. This will invalidate - non-generated endpoints - */ - Set requiredNames = requiredZones.stream() + Set wantedNames = requiredZones.stream() .flatMap(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), - deploymentSpec) + deploymentSpec, generatedId, true) .stream()) .collect(Collectors.toCollection(LinkedHashSet::new)); - // Preserve any currently present names that are still valid - List currentNames = currentCert.map(EndpointCertificate::requestedDnsSans) - .orElseGet(List::of); - zonesInSystem.stream() - .map(zone -> controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), deploymentSpec)) - .filter(currentNames::containsAll) - .forEach(requiredNames::addAll); + // Preserve any currently present names that are still valid (i.e. the name points to a zone found in this system) + Set currentNames = current.map(EndpointCertificate::requestedDnsSans) + .map(Set::copyOf) + .orElseGet(Set::of); + for (var zone : zonesInSystem) { + List wantedNamesZone = controller.routing().certificateDnsNames(new DeploymentId(deployment.applicationId(), zone), + deploymentSpec, + generatedId, + true); + if (currentNames.containsAll(wantedNamesZone)) { + wantedNames.addAll(wantedNamesZone); + } + } - log.log(Level.INFO, String.format("Requesting new endpoint certificate from Cameo for application %s", deployment.applicationId().serializedForm())); - String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); + // Request certificate + LOG.log(Level.INFO, String.format("Requesting new endpoint certificate for application %s", deployment.applicationId().serializedForm())); + String algo = endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); boolean useAlternativeProvider = useAlternateCertProvider.with(FetchVector.Dimension.INSTANCE_ID, deployment.applicationId().serializedForm()).value(); String keyPrefix = deployment.applicationId().toFullString(); - var t0 = Instant.now(); - EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(requiredNames), currentCert, algo, useAlternativeProvider); - var t1 = Instant.now(); - log.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); - return endpointCertificate; + Instant t0 = controller.clock().instant(); + EndpointCertificate endpointCertificate = certificateProvider.requestCaSignedCertificate(keyPrefix, List.copyOf(wantedNames), current, algo, useAlternativeProvider); + Instant t1 = controller.clock().instant(); + LOG.log(Level.INFO, String.format("Endpoint certificate request for application %s returned after %s", deployment.applicationId().serializedForm(), Duration.between(t0, t1))); + return endpointCertificate.withGeneratedId(generatedId); } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java index f4936dcfa8b..88041f1c684 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -3,32 +3,24 @@ package com.yahoo.vespa.hosted.controller.maintenance; import com.google.common.collect.Sets; import com.yahoo.component.annotation.Inject; -import com.yahoo.config.application.api.DeploymentSpec; import com.yahoo.config.provision.ApplicationId; import com.yahoo.config.provision.InstanceName; import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.transaction.Mutex; import com.yahoo.transaction.NestedTransaction; -import com.yahoo.vespa.flags.BooleanFlag; -import com.yahoo.vespa.flags.Flags; -import com.yahoo.vespa.flags.IntFlag; -import com.yahoo.vespa.flags.PermanentFlags; -import com.yahoo.vespa.flags.StringFlag; import com.yahoo.vespa.hosted.controller.Application; import com.yahoo.vespa.hosted.controller.Controller; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest; -import com.yahoo.vespa.hosted.controller.application.Endpoint; -import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint; -import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager; import com.yahoo.vespa.hosted.controller.application.Deployment; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; +import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate; import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; @@ -42,11 +34,9 @@ import java.util.HashSet; import java.util.List; import java.util.Optional; import java.util.OptionalInt; -import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; -import java.util.stream.Stream; /** * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates. @@ -66,9 +56,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { private final EndpointSecretManager endpointSecretManager; private final EndpointCertificateProvider endpointCertificateProvider; final Comparator oldestFirst = Comparator.comparing(e -> e.deployment.at()); - private final StringFlag endpointCertificateAlgo; - private final BooleanFlag useAlternateCertProvider; - private final IntFlag assignRandomizedIdRate; @Inject public EndpointCertificateMaintainer(Controller controller, Duration interval) { @@ -79,9 +66,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { this.endpointSecretManager = controller.serviceRegistry().secretManager(); this.curator = controller().curator(); this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); - this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource()); - this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource()); - this.assignRandomizedIdRate = Flags.ASSIGNED_RANDOMIZED_ID_RATE.bindTo(controller.flagSource()); } @Override @@ -92,12 +76,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { updateRefreshedCertificates(); deleteUnusedCertificates(); deleteOrReportUnmanagedCertificates(); - assignRandomizedIds(); } catch (Exception e) { log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e); return 1.0; } - return 0.0; } @@ -269,115 +251,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer { } } - private void assignRandomizedIds() { - List assignedCertificates = curator.readAssignedCertificates(); - /* - only assign randomized id if: - * instance is present - * randomized id is not already assigned - * feature flag is enabled - */ - assignedCertificates.stream() - .filter(c -> c.instance().isPresent()) - .filter(c -> c.certificate().generatedId().isEmpty()) - .filter(c -> controller().applications().getApplication(c.application()).isPresent()) // In case application has been deleted, but certificate is pending deletion - .limit(assignRandomizedIdRate.value()) - .forEach(c -> assignRandomizedId(c.application(), c.instance().get())); - } - - /* - Assign randomized id according to these rules: - * Instance is not mentioned in the deployment spec for this application - -> assume this is a manual deployment. Assign a randomized id to the certificate, save using instance only - * Instance is mentioned in deployment spec: - -> If there is a random endpoint assigned to tenant:application -> use this also for the "instance" certificate - -> Otherwise assign a random endpoint and write to the application and the instance. - */ - private void assignRandomizedId(TenantAndApplicationId tenantAndApplicationId, InstanceName instanceName) { - Optional assignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.of(instanceName)); - if (assignedCertificate.isEmpty()) { - log.log(Level.INFO, "Assigned certificate missing for " + tenantAndApplicationId.instance(instanceName).toFullString() + " when assigning randomized id"); - } - // Verify that the assigned certificate still does not have randomized id assigned - if (assignedCertificate.get().certificate().generatedId().isPresent()) return; - - controller().applications().lockApplicationOrThrow(tenantAndApplicationId, application -> { - DeploymentSpec deploymentSpec = application.get().deploymentSpec(); - if (deploymentSpec.instance(instanceName).isPresent()) { - Optional applicationLevelAssignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.empty()); - assignApplicationRandomId(assignedCertificate.get(), applicationLevelAssignedCertificate); - } else { - assignInstanceRandomId(assignedCertificate.get()); - } - }); - } - - private void assignApplicationRandomId(AssignedCertificate instanceLevelAssignedCertificate, Optional applicationLevelAssignedCertificate) { - TenantAndApplicationId tenantAndApplicationId = instanceLevelAssignedCertificate.application(); - if (applicationLevelAssignedCertificate.isPresent()) { - // Application level assigned certificate with randomized id already exists. Copy randomized id to instance level certificate and request with random names. - EndpointCertificate withRandomNames = requestRandomNames( - tenantAndApplicationId, - instanceLevelAssignedCertificate.instance(), - applicationLevelAssignedCertificate.get().certificate().generatedId() - .orElseThrow(() -> new IllegalArgumentException("Application certificate already assigned to " + tenantAndApplicationId.toString() + ", but random id is missing")), - Optional.of(instanceLevelAssignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = instanceLevelAssignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } else { - // No application level certificate exists, generate new assigned certificate with the randomized id based names only, then request same names also for instance level cert - String randomId = generateRandomId(); - EndpointCertificate applicationLevelEndpointCert = requestRandomNames(tenantAndApplicationId, Optional.empty(), randomId, Optional.empty()); - AssignedCertificate applicationLevelCert = new AssignedCertificate(tenantAndApplicationId, Optional.empty(), applicationLevelEndpointCert); - - EndpointCertificate instanceLevelEndpointCert = requestRandomNames(tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), randomId, Optional.of(instanceLevelAssignedCertificate.certificate())); - instanceLevelAssignedCertificate = instanceLevelAssignedCertificate.with(instanceLevelEndpointCert); - - // Save both in transaction - try (NestedTransaction transaction = new NestedTransaction()) { - curator.writeAssignedCertificate(instanceLevelAssignedCertificate, transaction); - curator.writeAssignedCertificate(applicationLevelCert, transaction); - transaction.commit(); - } - } - } - - private void assignInstanceRandomId(AssignedCertificate assignedCertificate) { - String randomId = generateRandomId(); - EndpointCertificate withRandomNames = requestRandomNames(assignedCertificate.application(), assignedCertificate.instance(), randomId, Optional.of(assignedCertificate.certificate())); - AssignedCertificate assignedCertWithRandomNames = assignedCertificate.with(withRandomNames); - curator.writeAssignedCertificate(assignedCertWithRandomNames); - } - - private EndpointCertificate requestRandomNames(TenantAndApplicationId tenantAndApplicationId, Optional instanceName, String randomId, Optional previousRequest) { - String dnsSuffix = Endpoint.dnsSuffix(controller().system()); - List newSanDnsEntries = List.of( - "*.%s.z%s".formatted(randomId, dnsSuffix), - "*.%s.g%s".formatted(randomId, dnsSuffix), - "*.%s.a%s".formatted(randomId, dnsSuffix)); - List existingSanDnsEntries = previousRequest.map(EndpointCertificate::requestedDnsSans).orElse(List.of()); - List requestNames = Stream.concat(existingSanDnsEntries.stream(), newSanDnsEntries.stream()).toList(); - String key = instanceName.map(tenantAndApplicationId::instance).map(ApplicationId::toFullString).orElseGet(tenantAndApplicationId::toString); - return endpointCertificateProvider.requestCaSignedCertificate( - key, - requestNames, - previousRequest, - endpointCertificateAlgo.value(), - useAlternateCertProvider.value()) - .withGeneratedId(randomId); - } - - private String generateRandomId() { - List unassignedIds = curator.readUnassignedCertificates().stream().map(UnassignedCertificate::id).toList(); - List assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::generatedId).filter(Optional::isPresent).map(Optional::get).toList(); - Set allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet()); - String randomId; - do { - randomId = GeneratedEndpoint.createPart(controller().random(true)); - } while (allIds.contains(randomId)); - return randomId; - } - private static String asString(TenantAndApplicationId application, Optional instanceName) { return application.toString() + instanceName.map(name -> "." + name.value()).orElse(""); } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java index dc9c4650191..f4a70e11e4d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java @@ -6,7 +6,6 @@ import com.yahoo.component.Version; import com.yahoo.component.annotation.Inject; import com.yahoo.concurrent.UncheckedTimeoutException; import com.yahoo.config.provision.ApplicationId; -import com.yahoo.config.provision.ClusterSpec; import com.yahoo.config.provision.ClusterSpec.Id; import com.yahoo.config.provision.HostName; import com.yahoo.config.provision.InstanceName; @@ -643,6 +642,10 @@ public class CuratorDb { curator.delete(endpointCertificatePath(application, instanceName)); } + public void removeAssignedCertificate(TenantAndApplicationId application, Optional instanceName, NestedTransaction transaction) { + transaction.add(CuratorTransaction.from(CuratorOperations.delete(endpointCertificatePath(application, instanceName).getAbsolute()), curator)); + } + // TODO(mpolden): Remove this. Caller should make an explicit decision to read certificate for a particular instance public Optional readAssignedCertificate(ApplicationId applicationId) { return readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())); @@ -651,7 +654,7 @@ public class CuratorDb { public Optional readAssignedCertificate(TenantAndApplicationId application, Optional instance) { return readSlime(endpointCertificatePath(application, instance)).map(Slime::get) .map(EndpointCertificateSerializer::fromSlime) - .map(cert -> new AssignedCertificate(application, instance, cert)); + .map(cert -> new AssignedCertificate(application, instance, cert, false)); } public List readAssignedCertificates() { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java index 912bd051a31..8eddf8f4d45 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/certificate/EndpointCertificatesHandler.java @@ -19,6 +19,7 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.certificate.AssignedCertificate; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; import com.yahoo.vespa.hosted.controller.persistence.EndpointCertificateSerializer; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import java.util.List; import java.util.Optional; @@ -73,11 +74,11 @@ public class EndpointCertificatesHandler extends ThreadedHttpRequestHandler { public StringResponse reRequestEndpointCertificateFor(String instanceId, boolean ignoreExisting) { ApplicationId applicationId = ApplicationId.fromFullString(instanceId); - if (controller.routing().generatedEndpointsEnabled(applicationId)) { + if (controller.routing().endpointConfig(applicationId) == EndpointConfig.generated) { throw new IllegalArgumentException("Cannot re-request certificate. " + instanceId + " is assigned certificate from a pool"); } try (var lock = curator.lock(TenantAndApplicationId.from(applicationId))) { - AssignedCertificate assignedCertificate = curator.readAssignedCertificate(applicationId) + AssignedCertificate assignedCertificate = curator.readAssignedCertificate(TenantAndApplicationId.from(applicationId), Optional.of(applicationId.instance())) .orElseThrow(() -> new RestApiException.NotFound("No certificate found for application " + applicationId.serializedForm())); String algo = this.endpointCertificateAlgo.with(FetchVector.Dimension.INSTANCE_ID, applicationId.serializedForm()).value(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java new file mode 100644 index 00000000000..555fd024e47 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/EndpointConfig.java @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.routing; + +/** + * Endpoint configurations supported for an application. + * + * @author mpolden + */ +public enum EndpointConfig { + + /** Only legacy endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ + legacy, + + /** Legacy and generated endpoints will be published in DNS. Certificate will contain both legacy and generated names, and is never assigned from a pool */ + combined, + + /** Only generated endpoints will be published in DNS. Certificate will contain generated names only. Certificate is assigned from a pool */ + generated; + + /** Returns whether this config supports legacy endpoints */ + public boolean supportsLegacy() { + return this == legacy || this == combined; + } + + /** Returns whether this config supports generated endpoints */ + public boolean supportsGenerated() { + return this == combined || this == generated; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java index 273f8fd8838..5bfd50ce76d 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/PreparedEndpoints.java @@ -27,14 +27,13 @@ import java.util.stream.Collectors; public record PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List rotations, - Optional certificate) { + EndpointCertificate certificate) { - public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List rotations, Optional certificate) { + public PreparedEndpoints(DeploymentId deployment, EndpointList endpoints, List rotations, EndpointCertificate certificate) { this.deployment = Objects.requireNonNull(deployment); this.endpoints = Objects.requireNonNull(endpoints); this.rotations = List.copyOf(Objects.requireNonNull(rotations)); - this.certificate = Objects.requireNonNull(certificate); - certificate.ifPresent(endpointCertificate -> requireMatchingSans(endpointCertificate, endpoints)); + this.certificate = requireMatchingSans(certificate, endpoints); } /** Returns the endpoints contained in this as {@link com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint} */ @@ -101,13 +100,15 @@ public record PreparedEndpoints(DeploymentId deployment, }; } - private static void requireMatchingSans(EndpointCertificate certificate, EndpointList endpoints) { + private static EndpointCertificate requireMatchingSans(EndpointCertificate certificate, EndpointList endpoints) { + Objects.requireNonNull(certificate); for (var endpoint : endpoints.not().scope(Endpoint.Scope.weighted)) { // Weighted endpoints are not present in certificate if (!certificate.sanMatches(endpoint.dnsName())) { throw new IllegalArgumentException(endpoint + " has no matching SAN. Certificate contains " + certificate.requestedDnsSans()); } } + return certificate; } } diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java index 99f60735f6e..c94a8fe1cdf 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/context/DeploymentRoutingContext.java @@ -45,7 +45,7 @@ public abstract class DeploymentRoutingContext implements RoutingContext { * * @return the container endpoints relevant for this deployment, as declared in deployment spec */ - public final PreparedEndpoints prepare(BasicServicesXml services, Optional certificate, LockedApplication application) { + public final PreparedEndpoints prepare(BasicServicesXml services, EndpointCertificate certificate, LockedApplication application) { return routing.prepare(deployment, services, certificate, application); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java index 76ff6ced599..25e13e5950c 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java @@ -73,7 +73,6 @@ import java.util.Set; import java.util.TreeSet; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static com.yahoo.config.provision.SystemName.main; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devAwsUsEast2a; @@ -952,8 +951,6 @@ public class ControllerTest { // Create app1 var context1 = tester.newDeploymentContext("tenant1", "app1", "default"); var prodZone = ZoneId.from("prod", "us-west-1"); - var stagingZone = ZoneId.from("staging", "us-east-3"); - var testZone = ZoneId.from("test", "us-east-1"); tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone)); var applicationPackage = new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service")) .region(prodZone.region()) @@ -962,16 +959,23 @@ public class ControllerTest { context1.submit(applicationPackage).deploy(); var cert = certificate.apply(context1.instance()); assertTrue(cert.isPresent(), "Provisions certificate in " + Environment.prod); - assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud", - "app1.tenant1.global.vespa.oath.cloud", - "*.app1.tenant1.global.vespa.oath.cloud"), - Stream.of(prodZone, testZone, stagingZone) - .flatMap(zone -> Stream.of("", "*.") - .map(prefix -> prefix + "app1.tenant1." + zone.region().value() + - (zone.environment() == Environment.prod ? "" : "." + zone.environment().value()) + - ".vespa.oath.cloud"))) - .collect(Collectors.toUnmodifiableSet()), - Set.copyOf(tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(cert.get().rootRequestId()))); + assertEquals(List.of("*.app1.tenant1.global.vespa.oath.cloud", + "*.app1.tenant1.us-east-1.test.vespa.oath.cloud", + "*.app1.tenant1.us-east-3.staging.vespa.oath.cloud", + "*.app1.tenant1.us-west-1.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "app1.tenant1.global.vespa.oath.cloud", + "app1.tenant1.us-east-1.test.vespa.oath.cloud", + "app1.tenant1.us-east-3.staging.vespa.oath.cloud", + "app1.tenant1.us-west-1.vespa.oath.cloud", + "vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud"), + tester.controllerTester().serviceRegistry().endpointCertificateMock() + .dnsNamesOf(cert.get().rootRequestId()) + .stream() + .sorted() + .toList()); // Next deployment reuses certificate context1.submit(applicationPackage).deploy(); diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java index 2bc11adddf7..b5dac883d7b 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java @@ -17,18 +17,21 @@ import com.yahoo.security.SignatureAlgorithm; import com.yahoo.security.X509CertificateBuilder; import com.yahoo.security.X509CertificateUtils; import com.yahoo.test.ManualClock; +import com.yahoo.transaction.Mutex; import com.yahoo.vespa.flags.Flags; import com.yahoo.vespa.hosted.controller.ControllerTester; -import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProviderMock; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidatorImpl; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateValidatorMock; import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage; import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock; import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; +import com.yahoo.vespa.hosted.controller.routing.EndpointConfig; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -58,12 +61,13 @@ public class EndpointCertificatesTest { private final ControllerTester tester = new ControllerTester(); private final SecretStoreMock secretStore = new SecretStoreMock(); - private final CuratorDb mockCuratorDb = tester.curator(); + private final CuratorDb curator = tester.curator(); private final ManualClock clock = tester.clock(); private final EndpointCertificateProviderMock endpointCertificateProviderMock = new EndpointCertificateProviderMock(); private final EndpointCertificateValidatorImpl endpointCertificateValidator = new EndpointCertificateValidatorImpl(secretStore, clock); private final EndpointCertificates endpointCertificates = new EndpointCertificates(tester.controller(), endpointCertificateProviderMock, endpointCertificateValidator); private final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192); + private final Mutex lock = () -> {}; private X509Certificate testCertificate; private X509Certificate testCertificate2; @@ -74,6 +78,9 @@ public class EndpointCertificatesTest { "*.default.default.global.vespa.oath.cloud", "default.default.aws-us-east-1a.vespa.oath.cloud", "*.default.default.aws-us-east-1a.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud", "default.default.us-east-1.test.vespa.oath.cloud", "*.default.default.us-east-1.test.vespa.oath.cloud", "default.default.us-east-3.staging.vespa.oath.cloud", @@ -93,7 +100,10 @@ public class EndpointCertificatesTest { private static final List expectedDevSans = List.of( "vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", "default.default.us-east-1.dev.vespa.oath.cloud", - "*.default.default.us-east-1.dev.vespa.oath.cloud" + "*.default.default.us-east-1.dev.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud" ); private X509Certificate makeTestCert(List sans) { @@ -108,7 +118,7 @@ public class EndpointCertificatesTest { return x509CertificateBuilder.build(); } - private final Instance instance = new Instance(ApplicationId.defaultId()); + private final ApplicationId instance = ApplicationId.defaultId(); private final String testKeyName = "testKeyName"; private final String testCertName = "testCertName"; private ZoneId prodZone; @@ -125,22 +135,20 @@ public class EndpointCertificatesTest { @Test void provisions_new_certificate_in_dev() { ZoneId testZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); - Optional cert = endpointCertificates.get(instance, testZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedDevSans, cert.get().requestedDnsSans()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, testZone), DeploymentSpec.empty, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedDevSans, cert.requestedDnsSans()); } @Test void provisions_new_certificate_in_prod() { - Optional cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedSans, cert.get().requestedDnsSans()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedSans, cert.requestedDnsSans()); } private ControllerTester publicTester() { @@ -160,66 +168,68 @@ public class EndpointCertificatesTest { "*.default.default.g.vespa-app.cloud", "default.default.aws-us-east-1a.z.vespa-app.cloud", "*.default.default.aws-us-east-1a.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud", "default.default.us-east-1.test.z.vespa-app.cloud", "*.default.default.us-east-1.test.z.vespa-app.cloud", "default.default.us-east-3.staging.z.vespa-app.cloud", "*.default.default.us-east-3.staging.z.vespa-app.cloud" ); - Optional cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedSans, cert.get().requestedDnsSans()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedSans, cert.requestedDnsSans()); } @Test void reuses_stored_certificate() { - mockCuratorDb.writeAssignedCertificate(assignedCertificate(instance.id(), new EndpointCertificate(testKeyName, testCertName, 7, 0, "request_id", Optional.of("leaf-request-uuid"), - List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + curator.writeAssignedCertificate(assignedCertificate(instance, new EndpointCertificate(testKeyName, testCertName, 7, 0, "request_id", Optional.of("leaf-request-uuid"), + List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", "default.default.global.vespa.oath.cloud", "*.default.default.global.vespa.oath.cloud", "default.default.aws-us-east-1a.vespa.oath.cloud", - "*.default.default.aws-us-east-1a.vespa.oath.cloud"), - "", Optional.empty(), Optional.empty(), Optional.empty()))); + "*.default.default.aws-us-east-1a.vespa.oath.cloud", + "*.f5549014.z.vespa.oath.cloud", + "*.f5549014.g.vespa.oath.cloud", + "*.f5549014.a.vespa.oath.cloud"), + "", Optional.empty(), Optional.empty(), Optional.empty()))); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7); - Optional cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertEquals(testKeyName, cert.get().keyName()); - assertEquals(testCertName, cert.get().certName()); - assertEquals(7, cert.get().version()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertEquals(testKeyName, cert.keyName()); + assertEquals(testCertName, cert.certName()); + assertEquals(7, cert.version()); } @Test void reprovisions_certificate_when_necessary() { - mockCuratorDb.writeAssignedCertificate(assignedCertificate(instance.id(), new EndpointCertificate(testKeyName, testCertName, -1, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()))); + curator.writeAssignedCertificate(assignedCertificate(instance, new EndpointCertificate(testKeyName, testCertName, -1, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()))); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0); - Optional cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertEquals(0, cert.get().version()); - assertEquals(cert, mockCuratorDb.readAssignedCertificate(instance.id()).map(AssignedCertificate::certificate)); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertEquals(0, cert.version()); + assertEquals(cert, curator.readAssignedCertificate(instance).map(AssignedCertificate::certificate).get()); } @Test void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { ZoneId testZone = ZoneId.from("prod.ap-northeast-1"); - mockCuratorDb.writeAssignedCertificate(assignedCertificate(instance.id(), new EndpointCertificate(testKeyName, testCertName, -1, 0, "original-request-uuid", Optional.of("leaf-request-uuid"), expectedSans, "mockCa", Optional.empty(), Optional.empty(), Optional.empty()))); + curator.writeAssignedCertificate(assignedCertificate(instance, new EndpointCertificate(testKeyName, testCertName, -1, 0, "original-request-uuid", Optional.of("leaf-request-uuid"), expectedSans, "mockCa", Optional.empty(), Optional.empty(), Optional.empty()))); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate2) + X509CertificateUtils.toPem(testCertificate2), 0); - Optional cert = endpointCertificates.get(instance, testZone, DeploymentSpec.empty); - assertTrue(cert.isPresent()); - assertEquals(0, cert.get().version()); - assertEquals(cert, mockCuratorDb.readAssignedCertificate(instance.id()).map(AssignedCertificate::certificate)); - assertEquals("original-request-uuid", cert.get().rootRequestId()); - assertNotEquals(Optional.of("leaf-request-uuid"), cert.get().leafRequestId()); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.get().requestedDnsSans())); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, testZone), DeploymentSpec.empty, lock); + assertEquals(0, cert.version()); + assertEquals(cert, curator.readAssignedCertificate(instance).map(AssignedCertificate::certificate).get()); + assertEquals("original-request-uuid", cert.rootRequestId()); + assertNotEquals(Optional.of("leaf-request-uuid"), cert.leafRequestId()); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.requestedDnsSans())); } @Test @@ -238,17 +248,16 @@ public class EndpointCertificatesTest { ); ZoneId testZone = tester.zoneRegistry().zones().all().in(Environment.staging).zones().stream().findFirst().orElseThrow().getId(); - Optional cert = endpointCertificates.get(instance, testZone, deploymentSpec); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.default.default.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.default.default.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.get().requestedDnsSans())); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, testZone), deploymentSpec, lock); + assertTrue(cert.keyName().matches("vespa.tls.default.default.*-key")); + assertTrue(cert.certName().matches("vespa.tls.default.default.*-cert")); + assertEquals(0, cert.version()); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(cert.requestedDnsSans())); } @Test void includes_application_endpoint_when_declared() { - Instance instance = new Instance(ApplicationId.from("t1", "a1", "default")); + ApplicationId instance = ApplicationId.from("t1", "a1", "default"); ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); ZoneId zone2 = ZoneId.from(Environment.prod, RegionName.from("aws-us-west-2a")); ControllerTester tester = publicTester(); @@ -280,28 +289,25 @@ public class EndpointCertificatesTest { "a1.t1.us-east-1.test.z.vespa-app.cloud", "*.a1.t1.us-east-1.test.z.vespa-app.cloud", "a1.t1.us-east-3.staging.z.vespa-app.cloud", - "*.a1.t1.us-east-3.staging.z.vespa-app.cloud" + "*.a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud" ).sorted().toList(); - Optional cert = endpointCertificates.get(instance, zone1, applicationPackage.deploymentSpec()); - assertTrue(cert.isPresent()); - assertTrue(cert.get().keyName().matches("vespa.tls.t1.a1.*-key")); - assertTrue(cert.get().certName().matches("vespa.tls.t1.a1.*-cert")); - assertEquals(0, cert.get().version()); - assertEquals(expectedSans, cert.get().requestedDnsSans().stream().sorted().toList()); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, zone1), applicationPackage.deploymentSpec(), lock); + assertTrue(cert.keyName().matches("vespa.tls.t1.a1.*-key")); + assertTrue(cert.certName().matches("vespa.tls.t1.a1.*-cert")); + assertEquals(0, cert.version()); + assertEquals(expectedSans, cert.requestedDnsSans().stream().sorted().toList()); } @Test public void assign_certificate_from_pool() { - // Initial certificate is requested directly from provider - Optional certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(certFromProvider.isPresent()); - assertFalse(certFromProvider.get().generatedId().isPresent()); - - // Pooled certificates become available tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); try { - addCertificateToPool("pool-cert-1", UnassignedCertificate.State.requested); - endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); + addCertificateToPool("bad0f00d", UnassignedCertificate.State.requested, tester); + endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); fail("Expected exception as certificate is not ready"); } catch (IllegalArgumentException ignored) {} @@ -311,76 +317,169 @@ public class EndpointCertificatesTest { // Certificate is assigned from pool instead. The previously assigned certificate will eventually be cleaned up // by EndpointCertificateMaintainer { // prod - String certId = "pool-cert-1"; - addCertificateToPool(certId, UnassignedCertificate.State.ready); - Optional cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertEquals(certId, cert.get().generatedId().get()); - assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance.id()), Optional.empty()).get().certificate().generatedId().get(), "Certificate is assigned at application-level"); + String certId = "bad0f00d"; + addCertificateToPool(certId, UnassignedCertificate.State.ready, tester); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, prodZone), DeploymentSpec.empty, lock); + assertEquals(certId, cert.generatedId().get()); + assertEquals(certId, tester.curator().readAssignedCertificate(TenantAndApplicationId.from(instance), Optional.empty()).get().certificate().generatedId().get(), "Certificate is assigned at application-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); - assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); + assertEquals(clock.instant().getEpochSecond(), cert.lastRequested()); } { // dev - String certId = "pool-cert-2"; - addCertificateToPool(certId, UnassignedCertificate.State.ready); + String certId = "f00d0bad"; + addCertificateToPool(certId, UnassignedCertificate.State.ready, tester); ZoneId devZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.dev).zones().stream().findFirst().orElseThrow().getId(); - Optional cert = endpointCertificates.get(instance, devZone, DeploymentSpec.empty); - assertEquals(certId, cert.get().generatedId().get()); - assertEquals(certId, tester.curator().readAssignedCertificate(instance.id()).get().certificate().generatedId().get(), "Certificate is assigned at instance-level"); + EndpointCertificate cert = endpointCertificates.get(new DeploymentId(instance, devZone), DeploymentSpec.empty, lock); + assertEquals(certId, cert.generatedId().get()); + assertEquals(certId, tester.curator().readAssignedCertificate(instance).get().certificate().generatedId().get(), "Certificate is assigned at instance-level"); assertTrue(tester.controller().curator().readUnassignedCertificate(certId).isEmpty(), "Certificate is removed from pool"); - assertEquals(clock.instant().getEpochSecond(), cert.get().lastRequested()); + assertEquals(clock.instant().getEpochSecond(), cert.lastRequested()); } } @Test - void reuse_per_instance_certificate_if_assigned_random_id() { - // Initial certificate is requested directly from provider - Optional certFromProvider = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertTrue(certFromProvider.isPresent()); - assertFalse(certFromProvider.get().generatedId().isPresent()); - - // Simulate endpoint certificate maintainer to assign random id - TenantAndApplicationId tenantAndApplicationId = TenantAndApplicationId.from(instance.id()); - Optional instanceName = Optional.of(instance.name()); - Optional assignedCertificate = tester.controller().curator().readAssignedCertificate(tenantAndApplicationId, instanceName); - assertTrue(assignedCertificate.isPresent()); - String assignedRandomId = "randomid"; - AssignedCertificate updated = assignedCertificate.get().with(assignedCertificate.get().certificate().withGeneratedId(assignedRandomId)); - tester.controller().curator().writeAssignedCertificate(updated); - - // Pooled certificates become available + public void certificate_migration() { + // An application is initially deployed with legacy config + ZoneId zone1 = ZoneId.from(Environment.prod, RegionName.from("aws-us-east-1c")); + ApplicationPackage applicationPackage = new ApplicationPackageBuilder().region(zone1.region()) + .build(); + ControllerTester tester = publicTester(); + EndpointCertificates endpointCertificates = new EndpointCertificates(tester.controller(), endpointCertificateProviderMock, new EndpointCertificateValidatorMock()); + ApplicationId instance = ApplicationId.from("t1", "a1", "default"); + DeploymentId deployment0 = new DeploymentId(instance, zone1); + final EndpointCertificate certificate = endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock); + final String generatedId = certificate.generatedId().get(); + assertEquals(List.of("vlfms2wpoa4nyrka2s5lktucypjtxkqhv.internal.vespa-app.cloud", + "a1.t1.g.vespa-app.cloud", + "*.a1.t1.g.vespa-app.cloud", + "a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud", + "a1.t1.us-east-1.test.z.vespa-app.cloud", + "*.a1.t1.us-east-1.test.z.vespa-app.cloud", + "a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.a1.t1.us-east-3.staging.z.vespa-app.cloud"), + certificate.requestedDnsSans()); + Optional assignedCertificate = tester.curator().readAssignedCertificate(deployment0.applicationId()); + assertTrue(assignedCertificate.isPresent(), "Certificate is assigned at instance level"); + assertTrue(assignedCertificate.get().certificate().generatedId().isPresent(), "Certificate contains generated ID"); + + // Re-requesting certificate does not make any changes, except last requested time + tester.clock().advance(Duration.ofHours(1)); + assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), + endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + "Next request returns same certificate and updates last requested time"); + + // An additional instance is added to deployment spec + applicationPackage = new ApplicationPackageBuilder().instances("default,beta") + .region(zone1.region()) + .build(); + DeploymentId deployment1 = new DeploymentId(ApplicationId.from("t1", "a1", "beta"), zone1); + EndpointCertificate betaCert = endpointCertificates.get(deployment1, applicationPackage.deploymentSpec(), lock); + assertEquals(List.of("v43ctkgqim52zsbwefrg6ixkuwidvsumy.internal.vespa-app.cloud", + "beta.a1.t1.g.vespa-app.cloud", + "*.beta.a1.t1.g.vespa-app.cloud", + "beta.a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.beta.a1.t1.aws-us-east-1c.z.vespa-app.cloud", + "*.f5549014.z.vespa-app.cloud", + "*.f5549014.g.vespa-app.cloud", + "*.f5549014.a.vespa-app.cloud", + "beta.a1.t1.us-east-1.test.z.vespa-app.cloud", + "*.beta.a1.t1.us-east-1.test.z.vespa-app.cloud", + "beta.a1.t1.us-east-3.staging.z.vespa-app.cloud", + "*.beta.a1.t1.us-east-3.staging.z.vespa-app.cloud"), + betaCert.requestedDnsSans()); + assertEquals(generatedId, betaCert.generatedId().get(), "Certificate inherits generated ID of existing instance"); + + // A dev instance is deployed + DeploymentId devDeployment0 = new DeploymentId(ApplicationId.from("t1", "a1", "dev"), + ZoneId.from("dev", "us-east-1")); + EndpointCertificate devCert0 = endpointCertificates.get(devDeployment0, applicationPackage.deploymentSpec(), lock); + assertNotEquals(generatedId, devCert0.generatedId().get(), "Dev deployments gets a new generated ID"); + assertEquals(List.of("vld3y4mggzpd5wmm5jmldzcbyetjoqtzq.internal.vespa-app.cloud", + "dev.a1.t1.us-east-1.dev.z.vespa-app.cloud", + "*.dev.a1.t1.us-east-1.dev.z.vespa-app.cloud", + "*.a89ff7c6.z.vespa-app.cloud", + "*.a89ff7c6.g.vespa-app.cloud", + "*.a89ff7c6.a.vespa-app.cloud"), + devCert0.requestedDnsSans()); + + // Application switches to combined config tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - - // Create 1 cert in pool - String certId = "pool-cert-1"; - addCertificateToPool(certId, UnassignedCertificate.State.ready); - - // Request cert for app - Optional cert = endpointCertificates.get(instance, prodZone, DeploymentSpec.empty); - assertEquals(assignedRandomId, cert.get().generatedId().get()); - - // Pooled cert remains unassigned - List unassignedCertificateIds = tester.curator().readUnassignedCertificates().stream() - .map(UnassignedCertificate::certificate) - .map(EndpointCertificate::generatedId) - .map(Optional::get) - .toList(); - assertEquals(List.of(certId), unassignedCertificateIds); + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true); + tester.clock().advance(Duration.ofHours(1)); + assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), + endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + "No change to certificate: Existing certificate is compatible with " + + EndpointConfig.combined + " config"); + assertTrue(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is assigned at instance level"); + assertFalse(tester.curator().readAssignedCertificate(TenantAndApplicationId.from(deployment0.applicationId()), Optional.empty()).isPresent(), + "Certificate is not assigned at application level"); + + // Application switches to generated config + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); + tester.clock().advance(Duration.ofHours(1)); + assertEquals(certificate.withLastRequested(tester.clock().instant().getEpochSecond()), + endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + "No change to certificate: Existing certificate is compatible with " + + EndpointConfig.generated + " config"); + assertFalse(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is no longer assigned at instance level"); + assertTrue(tester.curator().readAssignedCertificate(TenantAndApplicationId.from(deployment0.applicationId()), Optional.empty()).isPresent(), + "Certificate is assigned at application level"); + + // Both instances still use the same certificate + assertEquals(endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock), + endpointCertificates.get(deployment1, applicationPackage.deploymentSpec(), lock)); + + // Another dev instance is deployed, and is assigned certificate from pool + String poolCertId0 = "badf00d0"; + addCertificateToPool(poolCertId0, UnassignedCertificate.State.ready, tester); + EndpointCertificate devCert1 = endpointCertificates.get(new DeploymentId(ApplicationId.from("t1", "a1", "dev2"), + ZoneId.from("dev", "us-east-1")), + applicationPackage.deploymentSpec(), lock); + assertEquals(poolCertId0, devCert1.generatedId().get()); + + // Another application is deployed, and is assigned certificate from pool + String poolCertId1 = "badf00d1"; + addCertificateToPool(poolCertId1, UnassignedCertificate.State.ready, tester); + EndpointCertificate prodCertificate = endpointCertificates.get(new DeploymentId(ApplicationId.from("t1", "a2", "default"), + ZoneId.from("prod", "us-east-1")), + applicationPackage.deploymentSpec(), lock); + assertEquals(poolCertId1, prodCertificate.generatedId().get()); + + // Application switches back to legacy config + tester.flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), false); + tester.flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), true); + EndpointCertificate reissuedCertificate = endpointCertificates.get(deployment0, applicationPackage.deploymentSpec(), lock); + assertEquals(certificate.requestedDnsSans(), reissuedCertificate.requestedDnsSans()); + assertTrue(tester.curator().readAssignedCertificate(deployment0.applicationId()).isPresent(), "Certificate is assigned at instance level again"); + assertTrue(tester.curator().readAssignedCertificate(TenantAndApplicationId.from(deployment0.applicationId()), Optional.empty()).isPresent(), + "Certificate is still assigned at application level"); // Not removed because the assumption is that the application will eventually migrate back } - private void addCertificateToPool(String id, UnassignedCertificate.State state) { - EndpointCertificate cert = new EndpointCertificate(testKeyName, testCertName, 1, 0, + private void addCertificateToPool(String id, UnassignedCertificate.State state, ControllerTester tester) { + EndpointCertificate cert = new EndpointCertificate(testKeyName, + testCertName, + 1, + 0, "request-id", Optional.of("leaf-request-uuid"), - List.of("name1", "name2"), - "", Optional.empty(), - Optional.empty(), Optional.of(id)); + List.of("*." + id + ".z.vespa.oath.cloud", + "*." + id + ".g.vespa.oath.cloud", + "*." + id + ".a.vespa.oath.cloud"), + "", + Optional.empty(), + Optional.empty(), + Optional.of(id)); UnassignedCertificate pooledCert = new UnassignedCertificate(cert, state); tester.controller().curator().writeUnassignedCertificate(pooledCert); } private static AssignedCertificate assignedCertificate(ApplicationId instance, EndpointCertificate certificate) { - return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate); + return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate, false); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java index 2f996bac897..ee1a48c2812 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -45,7 +45,6 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.pro import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest; import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNotEquals; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -120,6 +119,7 @@ public class EndpointCertificateMaintainerTest { var applicationPackage = new ApplicationPackageBuilder() .region("us-west-1") + .container("default") .build(); DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); @@ -190,68 +190,6 @@ public class EndpointCertificateMaintainerTest { assertNotEquals(List.of(), endpointCertificateProvider.listCertificates()); } - @Test - void production_deployment_certificates_are_assigned_random_id() { - var app = ApplicationId.from("tenant", "app", "default"); - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty()); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - - maintainer.maintain(); - assertEquals(2, tester.curator().readAssignedCertificates().size()); - - // Verify random id is same for application and instance certificates - Optional applicationCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.empty()); - assertTrue(applicationCertificate.isPresent()); - Optional instanceCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.of(app.instance())); - assertTrue(instanceCertificate.isPresent()); - assertEquals(instanceCertificate.get().certificate().generatedId(), applicationCertificate.get().certificate().generatedId()); - - // Verify the 3 wildcard random names are same in all certs - List appWildcardSans = applicationCertificate.get().certificate().requestedDnsSans(); - assertEquals(3, appWildcardSans.size()); - List instanceSans = instanceCertificate.get().certificate().requestedDnsSans(); - List wildcards = instanceSans.stream().filter(appWildcardSans::contains).toList(); - assertEquals(appWildcardSans, wildcards); - } - - @Test - void existing_application_randomid_is_copied_to_new_instance_deployments() { - var instance1 = ApplicationId.from("tenant", "prod", "instance1"); - var instance2 = ApplicationId.from("tenant", "prod", "instance2"); - - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, instance1, List.of(systemTest, stagingTest,productionUsWest1),Optional.of("instance1")); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - maintainer.maintain(); - - String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().generatedId().get(); - - deployToAssignCert(deploymentTester, instance2, List.of(productionUsWest1), Optional.of("instance1,instance2")); - maintainer.maintain(); - assertEquals(3, tester.curator().readAssignedCertificates().size()); - - assertEquals(randomId, tester.curator().readAssignedCertificate(instance1).get().certificate().generatedId().get()); - } - - @Test - void dev_certificates_are_not_assigned_application_level_certificate() { - var devApp = ApplicationId.from("tenant", "devonly", "foo"); - DeploymentTester deploymentTester = new DeploymentTester(tester); - deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty()); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - List originalRequestedSans = tester.curator().readAssignedCertificate(devApp).get().certificate().requestedDnsSans(); - maintainer.maintain(); - assertEquals(1, tester.curator().readAssignedCertificates().size()); - - // Verify certificate is assigned random id and 3 new names - Optional assignedCertificate = tester.curator().readAssignedCertificate(devApp); - assertTrue(assignedCertificate.get().certificate().generatedId().isPresent()); - List newRequestedSans = assignedCertificate.get().certificate().requestedDnsSans(); - List randomizedNames = newRequestedSans.stream().filter(san -> !originalRequestedSans.contains(san)).toList(); - assertEquals(3, randomizedNames.size()); - } - @Test void deploy_to_other_manual_zone_refreshes_cert() { String devSan = "*.foo.manual.tenant.us-east-1.dev.vespa.oath.cloud"; @@ -301,10 +239,6 @@ public class EndpointCertificateMaintainerTest { Assertions.assertThat(usCentralWestSans).contains(centralSan); } - private void deploy() { - - } - private void deployToAssignCert(DeploymentTester tester, ApplicationId applicationId, List jobTypes, Optional instances) { var applicationPackageBuilder = new ApplicationPackageBuilder(); @@ -322,19 +256,15 @@ public class EndpointCertificateMaintainerTest { jobs.forEach(deploymentContext::runJob); } - EndpointCertificate certificate(List sans) { - return new EndpointCertificate("keyName", "certName", 0, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty()); - } - - private static AssignedCertificate assignedCertificate(ApplicationId instance, EndpointCertificate certificate) { - return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate); + return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate, false); } private void prepareCertificatePool(int numCertificates) { ((InMemoryFlagSource)tester.controller().flagSource()).withIntFlag(PermanentFlags.CERT_POOL_SIZE.id(), numCertificates); ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + ((InMemoryFlagSource)tester.controller().flagSource()).withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); // Provision certificates for (int i = 0; i < numCertificates; i++) { @@ -351,4 +281,5 @@ public class EndpointCertificateMaintainerTest { }); certificatePoolMaintainer.maintain(); } + } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java index 3405009714d..352b4e5f98e 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java @@ -1068,11 +1068,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void combined_endpoint_config() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.combined); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application int clustersPerZone = 2; @@ -1093,10 +1091,10 @@ public class RoutingPoliciesTest { // Deployment creates generated zone names List expectedRecords = List.of( // save me, jebus! - "a6414896.cafed00d.aws-eu-west-1.w.vespa-app.cloud", - "b36bf591.cafed00d.z.vespa-app.cloud", + "a6414896.f5549014.aws-eu-west-1.w.vespa-app.cloud", + "aa7591aa.f5549014.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", - "bc50b636.cafed00d.z.vespa-app.cloud", + "bc50b636.f5549014.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1.w.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1.w.vespa-app.cloud", @@ -1105,16 +1103,16 @@ public class RoutingPoliciesTest { "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "c33db5ed.cafed00d.z.vespa-app.cloud", - "d467800f.cafed00d.z.vespa-app.cloud", - "d71005bf.cafed00d.z.vespa-app.cloud", - "dd0971b4.cafed00d.z.vespa-app.cloud", - "eb48ad53.cafed00d.z.vespa-app.cloud", - "ec1e1288.cafed00d.z.vespa-app.cloud", - "f2fa41ec.cafed00d.g.vespa-app.cloud", - "f411d177.cafed00d.z.vespa-app.cloud", - "f4a4d111.cafed00d.a.vespa-app.cloud", - "fcf1bd63.cafed00d.aws-us-east-1.w.vespa-app.cloud", + "c33db5ed.f5549014.z.vespa-app.cloud", + "d467800f.f5549014.z.vespa-app.cloud", + "d71005bf.f5549014.z.vespa-app.cloud", + "dd0971b4.f5549014.g.vespa-app.cloud", + "eb48ad53.f5549014.z.vespa-app.cloud", + "ec1e1288.f5549014.z.vespa-app.cloud", + "f2fa41ec.f5549014.a.vespa-app.cloud", + "f411d177.f5549014.z.vespa-app.cloud", + "f4a4d111.f5549014.z.vespa-app.cloud", + "fcf1bd63.f5549014.aws-us-east-1.w.vespa-app.cloud", "foo.app1.tenant1.g.vespa-app.cloud" ); assertEquals(expectedRecords, tester.recordNames()); @@ -1178,23 +1176,23 @@ public class RoutingPoliciesTest { .build(); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(List.of( - "b36bf591.cafed00d.z.vespa-app.cloud", + "aa7591aa.f5549014.z.vespa-app.cloud", "bar.app1.tenant1.a.vespa-app.cloud", - "bc50b636.cafed00d.z.vespa-app.cloud", + "bc50b636.f5549014.z.vespa-app.cloud", "c0.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c1.app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1a.z.vespa-app.cloud", "c1.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "c33db5ed.cafed00d.z.vespa-app.cloud", - "d467800f.cafed00d.z.vespa-app.cloud", - "d71005bf.cafed00d.z.vespa-app.cloud", - "dd0971b4.cafed00d.z.vespa-app.cloud", - "eb48ad53.cafed00d.z.vespa-app.cloud", - "ec1e1288.cafed00d.z.vespa-app.cloud", - "f411d177.cafed00d.z.vespa-app.cloud", - "f4a4d111.cafed00d.a.vespa-app.cloud" + "c33db5ed.f5549014.z.vespa-app.cloud", + "d467800f.f5549014.z.vespa-app.cloud", + "d71005bf.f5549014.z.vespa-app.cloud", + "eb48ad53.f5549014.z.vespa-app.cloud", + "ec1e1288.f5549014.z.vespa-app.cloud", + "f2fa41ec.f5549014.a.vespa-app.cloud", + "f411d177.f5549014.z.vespa-app.cloud", + "f4a4d111.f5549014.z.vespa-app.cloud" ), tester.recordNames()); // Removing application removes all records @@ -1206,11 +1204,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints_enable_token() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void generated_endpoint_config_with_token() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.generated); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application without token @@ -1270,12 +1266,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints_only() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void generated_endpoint_config() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.generated); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - tester.controllerTester().flagSource() - .withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true) - .withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), false); addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application @@ -1317,12 +1310,10 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoints_multi_instance() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void combined_endpoint_config_with_multiple_instances() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.combined); var context0 = tester.newDeploymentContext("tenant1", "app1", "default"); var context1 = tester.newDeploymentContext("tenant1", "app1", "beta"); - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); - addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application int clustersPerZone = 1; @@ -1338,11 +1329,11 @@ public class RoutingPoliciesTest { tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1); context0.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(List.of("a0.app1.tenant1.a.vespa-app.cloud", - "a9c8c045.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c0.beta.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "e144a11b.cafed00d.z.vespa-app.cloud", - "ee82b867.cafed00d.a.vespa-app.cloud"), + "cbff1506.f5549014.z.vespa-app.cloud", + "e144a11b.f5549014.a.vespa-app.cloud", + "ee82b867.f5549014.z.vespa-app.cloud"), tester.recordNames()); tester.assertTargets(context0.application().id(), EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(context0.deploymentIdIn(zone1), 1, context1.deploymentIdIn(zone1), 1)); @@ -1356,11 +1347,11 @@ public class RoutingPoliciesTest { .build(); context0.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); assertEquals(List.of("a0.app1.tenant1.a.vespa-app.cloud", - "a9c8c045.cafed00d.z.vespa-app.cloud", "c0.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", "c0.beta.app1.tenant1.aws-us-east-1c.z.vespa-app.cloud", - "e144a11b.cafed00d.z.vespa-app.cloud", - "ee82b867.cafed00d.a.vespa-app.cloud"), + "cbff1506.f5549014.z.vespa-app.cloud", + "e144a11b.f5549014.a.vespa-app.cloud", + "ee82b867.f5549014.z.vespa-app.cloud"), tester.recordNames()); tester.assertTargets(context0.application().id(), EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0, Map.of(context1.deploymentIdIn(zone1), 1)); @@ -1374,10 +1365,9 @@ public class RoutingPoliciesTest { } @Test - public void generated_endpoint_migration_with_global_endpoint() { - var tester = new RoutingPoliciesTester(SystemName.Public); + public void migrate_legacy_to_combined_endpoint_config_with_global_endpoint() { + var tester = new RoutingPoliciesTester(SystemName.Public).setEndpointConfig(EndpointConfig.legacy); var context = tester.newDeploymentContext("tenant1", "app1", "default"); - addCertificateToPool("cafed00d", UnassignedCertificate.State.ready, tester); // Deploy application int clustersPerZone = 2; @@ -1392,8 +1382,8 @@ public class RoutingPoliciesTest { context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instanceId(), EndpointId.of("foo"), 0, zone1, zone2); - // Switch to generated - tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), true); + // Switch to combined + tester.setEndpointConfig(EndpointConfig.combined); context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy(); tester.assertTargets(context.instance().id(), EndpointId.of("foo"), ClusterSpec.Id.from("c0"), 0, Map.of(zone1, 1L, zone2, 1L), true); @@ -1403,9 +1393,13 @@ public class RoutingPoliciesTest { EndpointCertificate cert = new EndpointCertificate("testKey", "testCert", 1, 0, "request-id", Optional.of("leaf-request-uuid"), - List.of("name1", "name2"), - "", Optional.empty(), - Optional.empty(), Optional.of(id)); + List.of("*." + id + ".z.vespa-app.cloud", + "*." + id + ".g.vespa-app.cloud", + "*." + id + ".a.vespa-app.cloud"), + "", + Optional.empty(), + Optional.empty(), + Optional.of(id)); UnassignedCertificate pooledCert = new UnassignedCertificate(cert, state); tester.controllerTester().controller().curator().writeUnassignedCertificate(pooledCert); } @@ -1521,6 +1515,12 @@ public class RoutingPoliciesTest { .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); } + public RoutingPoliciesTester setEndpointConfig(EndpointConfig config) { + tester.controllerTester().flagSource().withBooleanFlag(Flags.LEGACY_ENDPOINTS.id(), config.supportsLegacy()); + tester.controllerTester().flagSource().withBooleanFlag(Flags.RANDOMIZED_ENDPOINT_NAMES.id(), config.supportsGenerated()); + return this; + } + public RoutingPolicies routingPolicies() { return tester.controllerTester().controller().routing().policies(); } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 27c9e9ee7da..8ecff84962f 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -429,6 +429,13 @@ public class Flags { "Takes effect at redeployment", APPLICATION_ID); + public static final UnboundStringFlag ENDPOINT_CONFIG = defineStringFlag( + "endpoint-config", "legacy", + List.of("mpolden", "tokle"), "2023-10-06", "2024-02-01", + "Set the endpoint config to use for an application. Must be 'legacy', 'combined' or 'generated'. See EndpointConfig for further details", + "Takes effect on next deployment through controller", + APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List owners, String createdAt, String expiresAt, String description, -- cgit v1.2.3