diff options
Diffstat (limited to 'controller-server/src')
11 files changed, 386 insertions, 86 deletions
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java index 81cba77b3c3..aa5f0ae0fdc 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java @@ -79,6 +79,7 @@ public class Controller extends AbstractComponent { private final Metric metric; private final RoutingController routingController; private final ControllerConfig controllerConfig; + private final SecretStore secretStore; /** * Creates a controller @@ -115,6 +116,7 @@ public class Controller extends AbstractComponent { auditLogger = new AuditLogger(curator, clock); jobControl = new JobControl(new JobControlFlags(curator, flagSource)); this.controllerConfig = controllerConfig; + this.secretStore = secretStore; // Record the version of this controller curator().writeControllerVersion(this.hostname(), ControllerVersion.CURRENT); @@ -281,6 +283,10 @@ public class Controller extends AbstractComponent { return metric; } + public SecretStore secretStore() { + return secretStore; + } + private Set<CloudName> clouds() { return zoneRegistry.zones().all().zones().stream() .map(ZoneApi::getCloudName) diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java index ac146858145..76fa52c0706 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java @@ -14,6 +14,7 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; import com.yahoo.container.jdisc.secretstore.SecretStore; import com.yahoo.security.SubjectAlternativeName; import com.yahoo.security.X509CertificateUtils; +import com.yahoo.vespa.curator.Lock; import com.yahoo.vespa.flags.BooleanFlag; import com.yahoo.vespa.flags.FetchVector; import com.yahoo.vespa.flags.FlagSource; @@ -67,6 +68,7 @@ public class EndpointCertificateManager { private final BooleanFlag validateEndpointCertificates; private final StringFlag deleteUnusedEndpointCertificates; private final BooleanFlag endpointCertInSharedRouting; + private final BooleanFlag useEndpointCertificateMaintainer; public EndpointCertificateManager(ZoneRegistry zoneRegistry, CuratorDb curator, @@ -81,6 +83,7 @@ public class EndpointCertificateManager { this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource); this.deleteUnusedEndpointCertificates = Flags.DELETE_UNUSED_ENDPOINT_CERTIFICATES.bindTo(flagSource); this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource); + this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(flagSource); Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> { try { this.deleteUnusedCertificates(); @@ -117,10 +120,9 @@ public class EndpointCertificateManager { return Optional.of(provisionedCertificateMetadata); } - // Reprovision certificate if it is missing SANs for the zone we are deploying to - var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans(); + // Re-provision certificate if it is missing SANs for the zone we are deploying to var requiredSansForZone = dnsNamesOf(instance.id(), zone); - if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) { + if (!currentCertificateMetadata.get().requestedDnsSans().containsAll(requiredSansForZone)) { var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec); curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata); // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry @@ -128,13 +130,17 @@ public class EndpointCertificateManager { return Optional.of(reprovisionedCertificateMetadata); } - // Look for and use refreshed certificate - var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); - if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { - var refreshedCertificateMetadata = currentCertificateMetadata.get().withVersion(latestAvailableVersion.getAsInt()); - validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); - curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); - return Optional.of(refreshedCertificateMetadata); + if (!useEndpointCertificateMaintainer.value()) { + // Look for and use refreshed certificate + var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get()); + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) { + var refreshedCertificateMetadata = currentCertificateMetadata.get() + .withVersion(latestAvailableVersion.getAsInt()) + .withLastRefreshed(clock.instant().getEpochSecond()); + validateEndpointCertificate(refreshedCertificateMetadata, instance, zone); + curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata); + return Optional.of(refreshedCertificateMetadata); + } } validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone); @@ -149,23 +155,31 @@ public class EndpointCertificateManager { private void deleteUnusedCertificates() { CleanupMode mode = CleanupMode.valueOf(deleteUnusedEndpointCertificates.value().toUpperCase()); - if (mode == CleanupMode.DISABLE) return; + if (mode == CleanupMode.DISABLE || useEndpointCertificateMaintainer.value()) return; var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS); curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { - log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() - + " has not been requested in a month and app has no deployments" - + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : "")); - if (mode == CleanupMode.ENABLE) { - endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); - curator.deleteEndpointCertificateMetadata(applicationId); + try (Lock lock = lock(applicationId)) { + if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { + log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + + " has not been requested in a month and app has no deployments" + + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : "")); + if (mode == CleanupMode.ENABLE) { + endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); + curator.deleteEndpointCertificateMetadata(applicationId); + } + } } } }); } + private Lock lock(ApplicationId applicationId) { + return curator.lock(TenantAndApplicationId.from(applicationId)); + } + private boolean hasNoDeployments(ApplicationId applicationId) { var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId)) .flatMap(app -> app.get(applicationId.instance())) @@ -187,8 +201,7 @@ public class EndpointCertificateManager { private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) { List<String> currentlyPresentNames = currentMetadata.isPresent() ? - currentMetadata.get().requestedDnsSans().orElseThrow(() -> new RuntimeException("Certificate metadata exists but SANs are not present!")) - : Collections.emptyList(); + currentMetadata.get().requestedDnsSans() : Collections.emptyList(); var requiredZones = new LinkedHashSet<>(Set.of(deploymentZone)); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java index ff2a1963967..56b7e5b2e46 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java @@ -63,6 +63,7 @@ public class ControllerMaintenance extends AbstractComponent { maintainers.add(new ContainerImageExpirer(controller, intervals.containerImageExpirer)); maintainers.add(new HostSwitchUpdater(controller, intervals.hostSwitchUpdater)); maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer)); + maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer)); } public Upgrader upgrader() { return upgrader; } @@ -109,6 +110,7 @@ public class ControllerMaintenance extends AbstractComponent { private final Duration containerImageExpirer; private final Duration hostSwitchUpdater; private final Duration reindexingTriggerer; + private final Duration endpointCertificateMaintainer; public Intervals(SystemName system) { this.system = Objects.requireNonNull(system); @@ -132,6 +134,7 @@ public class ControllerMaintenance extends AbstractComponent { this.containerImageExpirer = duration(2, HOURS); this.hostSwitchUpdater = duration(12, HOURS); this.reindexingTriggerer = duration(1, HOURS); + this.endpointCertificateMaintainer = duration(12, HOURS); } private Duration duration(long amount, TemporalUnit unit) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java new file mode 100644 index 00000000000..a1d7c3d16b4 --- /dev/null +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java @@ -0,0 +1,157 @@ +// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.google.common.collect.Sets; +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.config.provision.SystemName; +import com.yahoo.container.jdisc.secretstore.SecretNotFoundException; +import com.yahoo.container.jdisc.secretstore.SecretStore; +import com.yahoo.log.LogLevel; +import com.yahoo.vespa.curator.Lock; +import com.yahoo.vespa.flags.BooleanFlag; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.hosted.controller.Controller; +import com.yahoo.vespa.hosted.controller.Instance; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId; +import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType; +import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger; +import com.yahoo.vespa.hosted.controller.persistence.CuratorDb; + +import java.time.Clock; +import java.time.Duration; +import java.time.Instant; +import java.time.temporal.ChronoUnit; +import java.util.HashSet; +import java.util.Optional; +import java.util.OptionalInt; +import java.util.function.Predicate; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates + * + * @author andreer + */ +public class EndpointCertificateMaintainer extends ControllerMaintainer { + + private static final Logger log = Logger.getLogger(EndpointCertificateMaintainer.class.getName()); + + private final DeploymentTrigger deploymentTrigger; + private final Clock clock; + private final CuratorDb curator; + private final SecretStore secretStore; + private final EndpointCertificateProvider endpointCertificateProvider; + private final BooleanFlag useEndpointCertificateMaintainer; + + public EndpointCertificateMaintainer(Controller controller, Duration interval) { + super(controller, interval, null, SystemName.all()); + this.deploymentTrigger = controller.applications().deploymentTrigger(); + this.clock = controller.clock(); + this.secretStore = controller.secretStore(); + this.curator = controller().curator(); + this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider(); + this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(controller().flagSource()); + } + + @Override + protected boolean maintain() { + + if (!useEndpointCertificateMaintainer.value()) + return true; // handled by EndpointCertificateManager for now + + try { + // In order of importance + deployRefreshedCertificates(); + updateRefreshedCertificates(); + deleteUnusedCertificates(); + } catch (Exception e) { + log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e); + return false; + } + + return true; + } + + private void updateRefreshedCertificates() { + curator.readAllEndpointCertificateMetadata().forEach(((applicationId, endpointCertificateMetadata) -> { + // Look for and use refreshed certificate + var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata); + if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) { + var refreshedCertificateMetadata = endpointCertificateMetadata + .withVersion(latestAvailableVersion.getAsInt()) + .withLastRefreshed(clock.instant().getEpochSecond()); + try (Lock lock = lock(applicationId)) { + if (Optional.of(endpointCertificateMetadata).equals(curator.readEndpointCertificateMetadata(applicationId))) { + curator.writeEndpointCertificateMetadata(applicationId, refreshedCertificateMetadata); // Certificate not validated here, but on deploy. + } + } + } + })); + } + + /** + * If it's been a week since the cert has been refreshed, re-trigger all prod deployment jobs. + */ + private void deployRefreshedCertificates() { + var now = clock.instant(); + curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) -> + endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> { + Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime); + if (now.isAfter(refreshTime.plus(7, ChronoUnit.DAYS))) { + + controller().jobController().jobs(applicationId).forEach(job -> + controller().jobController().jobStatus(new JobId(applicationId, JobType.fromJobName(job.jobName()))).lastTriggered().ifPresent(run -> { + if (run.start().isBefore(refreshTime) && job.isProduction() && job.isDeployment()) { + deploymentTrigger.reTrigger(applicationId, job); + log.info("Re-triggering deployment job " + job.jobName() + " for instance " + + applicationId.serializedForm() + " to roll out refreshed endpoint certificate"); + } + })); + } + })); + } + + private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) { + try { + var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName())); + var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName())); + return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max(); + } catch (SecretNotFoundException s) { + return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version + } + } + + private void deleteUnusedCertificates() { + var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS); + curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> { + var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested()); + if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) { + try (Lock lock = lock(applicationId)) { + if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) { + log.log(Level.INFO, "Cert for app " + applicationId.serializedForm() + + " has not been requested in a month and app has no deployments, deleting from provider and ZK"); + endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData); + curator.deleteEndpointCertificateMetadata(applicationId); + } + } + } + }); + } + + private Lock lock(ApplicationId applicationId) { + return curator.lock(TenantAndApplicationId.from(applicationId)); + } + + private boolean hasNoDeployments(ApplicationId applicationId) { + var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId)) + .flatMap(app -> app.get(applicationId.instance())) + .map(Instance::deployments); + + return deployments.isEmpty() || deployments.get().size() == 0; + } + +} diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java index ba882ef7985..19f9542c679 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java @@ -7,7 +7,6 @@ import com.yahoo.slime.SlimeUtils; import com.yahoo.slime.Type; import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; -import java.util.List; import java.util.Optional; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -16,7 +15,7 @@ import java.util.stream.IntStream; * (de)serializes endpoint certificate metadata * <p> * A copy of package com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadata, - * but will soon be extended as we need to store some more information in the controller. + * but with additional fields as we need to store some more information in the controller. * * @author andreer */ @@ -36,6 +35,8 @@ public class EndpointCertificateMetadataSerializer { private final static String requestIdField = "requestId"; private final static String requestedDnsSansField = "requestedDnsSans"; private final static String issuerField = "issuer"; + private final static String expiryField = "expiry"; + private final static String lastRefreshedField = "lastRefreshed"; public static Slime toSlime(EndpointCertificateMetadata metadata) { Slime slime = new Slime(); @@ -44,13 +45,12 @@ public class EndpointCertificateMetadataSerializer { object.setString(certNameField, metadata.certName()); object.setLong(versionField, metadata.version()); object.setLong(lastRequestedField, metadata.lastRequested()); - - metadata.request_id().ifPresent(id -> object.setString(requestIdField, id)); - metadata.requestedDnsSans().ifPresent(sans -> { - Cursor cursor = object.setArray(requestedDnsSansField); - sans.forEach(cursor::addString); - }); - metadata.issuer().ifPresent(id -> object.setString(issuerField, id)); + object.setString(requestIdField, metadata.request_id()); + var cursor = object.setArray(requestedDnsSansField); + metadata.requestedDnsSans().forEach(cursor::addString); + object.setString(issuerField, metadata.issuer()); + metadata.expiry().ifPresent(expiry -> object.setLong(expiryField, expiry)); + metadata.lastRefreshed().ifPresent(refreshTime -> object.setLong(lastRefreshedField, refreshTime)); return slime; } @@ -58,32 +58,22 @@ public class EndpointCertificateMetadataSerializer { public static EndpointCertificateMetadata fromSlime(Inspector inspector) { if (inspector.type() != Type.OBJECT) throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!"); - Optional<String> request_id = inspector.field(requestIdField).valid() ? - Optional.of(inspector.field(requestIdField).asString()) : - Optional.empty(); - - Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ? - Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries()) - .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) : - Optional.empty(); - - Optional<String> issuer = inspector.field(issuerField).valid() ? - Optional.of(inspector.field(issuerField).asString()) : - Optional.empty(); - - long lastRequested = inspector.field(lastRequestedField).valid() ? - inspector.field(lastRequestedField).asLong() : - 1597200000L; // Wed Aug 12 02:40:00 UTC 2020 - // Not originally stored, so we default to when field was added return new EndpointCertificateMetadata( inspector.field(keyNameField).asString(), inspector.field(certNameField).asString(), Math.toIntExact(inspector.field(versionField).asLong()), - lastRequested, - request_id, - requestedDnsSans, - issuer); + inspector.field(lastRequestedField).asLong(), + inspector.field(requestIdField).asString(), + IntStream.range(0, inspector.field(requestedDnsSansField).entries()) + .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList()), + inspector.field(issuerField).asString(), + inspector.field(expiryField).valid() ? + Optional.of(inspector.field(expiryField).asLong()) : + Optional.empty(), + inspector.field(lastRefreshedField).valid() ? + Optional.of(inspector.field(lastRefreshedField).asLong()) : + Optional.empty()); } public static EndpointCertificateMetadata fromJsonString(String zkData) { diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java index cedae5b5a46..88191bc836b 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/billing/BillingApiHandler.java @@ -63,9 +63,8 @@ public class BillingApiHandler extends LoggingRequestHandler { private final TenantController tenantController; public BillingApiHandler(Executor executor, - AccessLog accessLog, Controller controller) { - super(executor, accessLog); + super(executor); this.billingController = controller.serviceRegistry().billingController(); this.applicationController = controller.applications(); this.tenantController = controller.tenants(); diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java index 43a03bf10f3..129f1e109df 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsHandler.java @@ -5,8 +5,6 @@ import com.google.inject.Inject; import com.yahoo.container.jdisc.HttpRequest; import com.yahoo.container.jdisc.HttpResponse; import com.yahoo.container.jdisc.LoggingRequestHandler; -import com.yahoo.container.logging.AccessLog; -import java.util.logging.Level; import com.yahoo.restapi.ErrorResponse; import com.yahoo.restapi.JacksonJsonResponse; import com.yahoo.restapi.Path; @@ -16,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.systemflags.v1.FlagsTarget; import com.yahoo.vespa.hosted.controller.api.systemflags.v1.SystemFlagsDataArchive; import java.util.concurrent.Executor; +import java.util.logging.Level; /** * Handler implementation for '/system-flags/v1', an API for controlling system-wide feature flags @@ -32,9 +31,8 @@ public class SystemFlagsHandler extends LoggingRequestHandler { @Inject public SystemFlagsHandler(ZoneRegistry zoneRegistry, ServiceIdentityProvider identityProvider, - Executor executor, - AccessLog accessLog) { - super(executor, accessLog); + Executor executor) { + super(executor); this.deployer = new SystemFlagsDeployer(identityProvider, zoneRegistry.system(), FlagsTarget.getAllTargetsInSystem(zoneRegistry)); } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java index 1e3771dedb0..4f2000b1902 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java @@ -114,7 +114,7 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); - assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()); + assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans()); } @Test @@ -124,12 +124,18 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); - assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()); + assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans()); } @Test public void reuses_stored_certificate_metadata() { - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0)); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id", + List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + "default.default.global.vespa.oath.cloud", + "*.default.default.global.vespa.oath.cloud", + "default.default.aws-us-east-1a.vespa.oath.cloud", + "*.default.default.aws-us-east-1a.vespa.oath.cloud"), + "", Optional.empty(), Optional.empty())); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7); secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); @@ -146,7 +152,13 @@ public class EndpointCertificateManagerTest { secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8); secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9); secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8); - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0)); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id", + List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud", + "default.default.global.vespa.oath.cloud", + "*.default.default.global.vespa.oath.cloud", + "default.default.aws-us-east-1a.vespa.oath.cloud", + "*.default.default.aws-us-east-1a.vespa.oath.cloud"), + "issuer", Optional.empty(), Optional.empty())); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(testKeyName, endpointCertificateMetadata.get().keyName()); @@ -156,7 +168,7 @@ public class EndpointCertificateManagerTest { @Test public void reprovisions_certificate_when_necessary() { - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, Optional.of("uuid"), Optional.of(List.of()), Optional.empty())); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0); Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty()); @@ -169,7 +181,7 @@ public class EndpointCertificateManagerTest { public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() { ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId(); - mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, Optional.of("uuid"), Optional.of(expectedSans), Optional.of("mockCa"))); + mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", expectedSans, "mockCa", Optional.empty(), Optional.empty())); secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1); secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1); @@ -180,7 +192,7 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.isPresent()); assertEquals(0, endpointCertificateMetadata.get().version()); assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id())); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow())); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans())); } @Test @@ -202,6 +214,6 @@ public class EndpointCertificateManagerTest { assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key")); assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert")); assertEquals(0, endpointCertificateMetadata.get().version()); - assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow())); + assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans())); } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java new file mode 100644 index 00000000000..dbf102f23d7 --- /dev/null +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java @@ -0,0 +1,126 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.vespa.hosted.controller.maintenance; + +import com.yahoo.config.provision.ApplicationId; +import com.yahoo.vespa.flags.Flags; +import com.yahoo.vespa.flags.InMemoryFlagSource; +import com.yahoo.vespa.hosted.controller.ControllerTester; +import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata; +import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext; +import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester; +import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock; +import org.junit.Before; +import org.junit.Test; + +import java.time.Duration; +import java.util.List; +import java.util.Optional; + +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest; +import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * @author andreer + */ +public class EndpointCertificateMaintainerTest { + + private final ControllerTester tester = new ControllerTester(); + private final SecretStoreMock secretStore = (SecretStoreMock) tester.controller().secretStore(); + private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1)); + private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty()); + + @Before + public void setUp() throws Exception { + ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.id(), true); + } + + @Test + public void old_and_unused_cert_is_deleted() { + tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata); + assertTrue(maintainer.maintain()); + assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty()); + } + + @Test + public void unused_but_recently_used_cert_is_not_deleted() { + EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); + tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); + assertTrue(maintainer.maintain()); + assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); + } + + @Test + public void refreshed_certificate_is_updated() { + EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond()); + tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert); + + secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); + secretStore.setSecret(exampleMetadata.certName(), "bar", 1); + + assertTrue(maintainer.maintain()); + + var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1)); + + assertEquals(updatedCert, tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId())); + } + + @Test + public void certificate_in_use_is_not_deleted() { + var appId = ApplicationId.from("tenant", "application", "default"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + + var applicationPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + + DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); + + deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); + + + tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata); + + assertTrue(maintainer.maintain()); + assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed! + } + + @Test + public void refreshed_certificate_is_deployed_after_one_week() { + var appId = ApplicationId.from("tenant", "application", "default"); + + DeploymentTester deploymentTester = new DeploymentTester(tester); + + var applicationPackage = new ApplicationPackageBuilder() + .region("us-west-1") + .build(); + + DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default"); + + deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1); + + tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata); + + assertTrue(maintainer.maintain()); + assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed! + + tester.clock().advance(Duration.ofDays(3)); + + secretStore.setSecret(exampleMetadata.keyName(), "foo", 1); + secretStore.setSecret(exampleMetadata.certName(), "bar", 1); + + maintainer.maintain(); + + tester.clock().advance(Duration.ofDays(8)); + + deploymentContext.assertNotRunning(productionUsWest1); + + maintainer.maintain(); + + deploymentContext.assertRunning(productionUsWest1); + } +} diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java index 9c6790f630b..00f5335bd82 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java @@ -10,46 +10,39 @@ import static org.junit.Assert.*; public class EndpointCertificateMetadataSerializerTest { - private final EndpointCertificateMetadata sample = - new EndpointCertificateMetadata("keyName", "certName", 1, 0); - private final EndpointCertificateMetadata sampleWithRequestMetadata = - new EndpointCertificateMetadata("keyName", "certName", 1, 0, Optional.of("requestId"), Optional.of(List.of("SAN1", "SAN2")), Optional.of("issuer")); + private final EndpointCertificateMetadata sampleWithExpiryAndLastRefreshed = + new EndpointCertificateMetadata("keyName", "certName", 1, 0, "requestId", List.of("SAN1", "SAN2"), "issuer", java.util.Optional.of(1628000000L), Optional.of(1612000000L)); + + private final EndpointCertificateMetadata sampleWithoutExpiry = + new EndpointCertificateMetadata("keyName", "certName", 1, 0, "requestId", List.of("SAN1", "SAN2"), "issuer", Optional.empty(), Optional.empty()); @Test - public void serialize() { + public void serializeWithExpiryAndLastRefreshed() { assertEquals( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0}", - EndpointCertificateMetadataSerializer.toSlime(sample).toString()); + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\",\"expiry\":1628000000,\"lastRefreshed\":1612000000}", + EndpointCertificateMetadataSerializer.toSlime(sampleWithExpiryAndLastRefreshed).toString()); } @Test - public void serializeWithRequestMetadata() { + public void serializeWithoutExpiryAndLastRefreshed() { assertEquals( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}", - EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString()); + EndpointCertificateMetadataSerializer.toSlime(sampleWithoutExpiry).toString()); } @Test - public void deserializeFromJson() { + public void deserializeFromJsonWithExpiryAndLastRefreshed() { assertEquals( - sample, + sampleWithExpiryAndLastRefreshed, EndpointCertificateMetadataSerializer.fromJsonString( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0}")); + "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\",\"expiry\":1628000000,\"lastRefreshed\":1612000000}")); } @Test - public void deserializeFromJsonWithRequestMetadata() { + public void deserializeFromJsonWithoutExpiryAndLastRefreshed() { assertEquals( - sampleWithRequestMetadata, + sampleWithoutExpiry, EndpointCertificateMetadataSerializer.fromJsonString( "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}")); } - - @Test - public void deserializeFromJsonWithDefaultLastRequested() { - assertEquals( - new EndpointCertificateMetadata("keyName", "certName", 1, 1597200000), - EndpointCertificateMetadataSerializer.fromJsonString( - "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}")); - } } diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json index 5af91e17bb7..6f67b0d8aa8 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json @@ -31,6 +31,9 @@ "name": "DeploymentMetricsMaintainer" }, { + "name": "EndpointCertificateMaintainer" + }, + { "name": "HostSwitchUpdater" }, { |