andreer/endpoint certificate maintainer (#16099)

* remove support for old formats and introduce EndpointCertificateMaintainer

* record certificate refresh time, run maintainer every 12 hours

* retrigger prod deployments if refreshed certificate not deployed after one week

* only re-trigger production jobs

* unit test EndpointCertificateMaintainer

* take application lock to avoid concurrent modifications when managing endpoint certs

* only trigger deployment jobs

Co-authored-by: Jon Marius Venstad <jonmv@users.noreply.github.com>

9 files changed, 382 insertions, 79 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
index 81cba77b3c3..aa5f0ae0fdc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/Controller.java
@@ -79,6 +79,7 @@ public class Controller extends AbstractComponent {
     private final Metric metric;
     private final RoutingController routingController;
     private final ControllerConfig controllerConfig;
+    private final SecretStore secretStore;
 
     /**
      * Creates a controller 
@@ -115,6 +116,7 @@ public class Controller extends AbstractComponent {
         auditLogger = new AuditLogger(curator, clock);
         jobControl = new JobControl(new JobControlFlags(curator, flagSource));
         this.controllerConfig = controllerConfig;
+        this.secretStore = secretStore;
 
         // Record the version of this controller
         curator().writeControllerVersion(this.hostname(), ControllerVersion.CURRENT);
@@ -281,6 +283,10 @@ public class Controller extends AbstractComponent {
         return metric;
     }
 
+    public SecretStore secretStore() {
+        return secretStore;
+    }
+
     private Set<CloudName> clouds() {
         return zoneRegistry.zones().all().zones().stream()
                            .map(ZoneApi::getCloudName)
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
index ac146858145..76fa52c0706 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
@@ -14,6 +14,7 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.security.SubjectAlternativeName;
 import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.BooleanFlag;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
@@ -67,6 +68,7 @@ public class EndpointCertificateManager {
     private final BooleanFlag validateEndpointCertificates;
     private final StringFlag deleteUnusedEndpointCertificates;
     private final BooleanFlag endpointCertInSharedRouting;
+    private final BooleanFlag useEndpointCertificateMaintainer;
 
     public EndpointCertificateManager(ZoneRegistry zoneRegistry,
                                       CuratorDb curator,
@@ -81,6 +83,7 @@ public class EndpointCertificateManager {
         this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource);
         this.deleteUnusedEndpointCertificates = Flags.DELETE_UNUSED_ENDPOINT_CERTIFICATES.bindTo(flagSource);
         this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource);
+        this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(flagSource);
         Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
             try {
                 this.deleteUnusedCertificates();
@@ -117,10 +120,9 @@ public class EndpointCertificateManager {
             return Optional.of(provisionedCertificateMetadata);
         }
 
-        // Reprovision certificate if it is missing SANs for the zone we are deploying to
-        var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans();
+        // Re-provision certificate if it is missing SANs for the zone we are deploying to
         var requiredSansForZone = dnsNamesOf(instance.id(), zone);
-        if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) {
+        if (!currentCertificateMetadata.get().requestedDnsSans().containsAll(requiredSansForZone)) {
             var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec);
             curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata);
             // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry
@@ -128,13 +130,17 @@ public class EndpointCertificateManager {
             return Optional.of(reprovisionedCertificateMetadata);
         }
 
-        // Look for and use refreshed certificate
-        var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get());
-        if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) {
-            var refreshedCertificateMetadata = currentCertificateMetadata.get().withVersion(latestAvailableVersion.getAsInt());
-            validateEndpointCertificate(refreshedCertificateMetadata, instance, zone);
-            curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata);
-            return Optional.of(refreshedCertificateMetadata);
+        if (!useEndpointCertificateMaintainer.value()) {
+            // Look for and use refreshed certificate
+            var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get());
+            if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) {
+                var refreshedCertificateMetadata = currentCertificateMetadata.get()
+                        .withVersion(latestAvailableVersion.getAsInt())
+                        .withLastRefreshed(clock.instant().getEpochSecond());
+                validateEndpointCertificate(refreshedCertificateMetadata, instance, zone);
+                curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata);
+                return Optional.of(refreshedCertificateMetadata);
+            }
         }
 
         validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone);
@@ -149,23 +155,31 @@ public class EndpointCertificateManager {
 
     private void deleteUnusedCertificates() {
         CleanupMode mode = CleanupMode.valueOf(deleteUnusedEndpointCertificates.value().toUpperCase());
-        if (mode == CleanupMode.DISABLE) return;
+        if (mode == CleanupMode.DISABLE || useEndpointCertificateMaintainer.value()) return;
 
         var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS);
         curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> {
             var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested());
             if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) {
-                log.log(Level.INFO, "Cert for app " + applicationId.serializedForm()
-                        + " has not been requested in a month and app has no deployments"
-                        + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : ""));
-                if (mode == CleanupMode.ENABLE) {
-                    endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData);
-                    curator.deleteEndpointCertificateMetadata(applicationId);
+                try (Lock lock = lock(applicationId)) {
+                    if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) {
+                        log.log(Level.INFO, "Cert for app " + applicationId.serializedForm()
+                                + " has not been requested in a month and app has no deployments"
+                                + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : ""));
+                        if (mode == CleanupMode.ENABLE) {
+                            endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData);
+                            curator.deleteEndpointCertificateMetadata(applicationId);
+                        }
+                    }
                 }
             }
         });
     }
 
+    private Lock lock(ApplicationId applicationId) {
+        return curator.lock(TenantAndApplicationId.from(applicationId));
+    }
+
     private boolean hasNoDeployments(ApplicationId applicationId) {
         var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId))
                 .flatMap(app -> app.get(applicationId.instance()))
@@ -187,8 +201,7 @@ public class EndpointCertificateManager {
     private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) {
 
         List<String> currentlyPresentNames = currentMetadata.isPresent() ?
-                currentMetadata.get().requestedDnsSans().orElseThrow(() -> new RuntimeException("Certificate metadata exists but SANs are not present!"))
-                : Collections.emptyList();
+                currentMetadata.get().requestedDnsSans() : Collections.emptyList();
 
         var requiredZones = new LinkedHashSet<>(Set.of(deploymentZone));
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index ff2a1963967..56b7e5b2e46 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -63,6 +63,7 @@ public class ControllerMaintenance extends AbstractComponent {
         maintainers.add(new ContainerImageExpirer(controller, intervals.containerImageExpirer));
         maintainers.add(new HostSwitchUpdater(controller, intervals.hostSwitchUpdater));
         maintainers.add(new ReindexingTriggerer(controller, intervals.reindexingTriggerer));
+        maintainers.add(new EndpointCertificateMaintainer(controller, intervals.endpointCertificateMaintainer));
     }
 
     public Upgrader upgrader() { return upgrader; }
@@ -109,6 +110,7 @@ public class ControllerMaintenance extends AbstractComponent {
         private final Duration containerImageExpirer;
         private final Duration hostSwitchUpdater;
         private final Duration reindexingTriggerer;
+        private final Duration endpointCertificateMaintainer;
 
         public Intervals(SystemName system) {
             this.system = Objects.requireNonNull(system);
@@ -132,6 +134,7 @@ public class ControllerMaintenance extends AbstractComponent {
             this.containerImageExpirer = duration(2, HOURS);
             this.hostSwitchUpdater = duration(12, HOURS);
             this.reindexingTriggerer = duration(1, HOURS);
+            this.endpointCertificateMaintainer = duration(12, HOURS);
         }
 
         private Duration duration(long amount, TemporalUnit unit) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
new file mode 100644
index 00000000000..cbb5363aff9
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -0,0 +1,157 @@
+// Copyright 2018 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.google.common.collect.Sets;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
+import com.yahoo.container.jdisc.secretstore.SecretStore;
+import com.yahoo.log.LogLevel;
+import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.hosted.controller.Controller;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
+import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
+
+import java.time.Clock;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.HashSet;
+import java.util.Optional;
+import java.util.OptionalInt;
+import java.util.function.Predicate;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates
+ *
+ * @author andreer
+ */
+public class EndpointCertificateMaintainer extends ControllerMaintainer {
+
+    private static final Logger log = Logger.getLogger(EndpointCertificateMaintainer.class.getName());
+
+    private final DeploymentTrigger deploymentTrigger;
+    private final Clock clock;
+    private final CuratorDb curator;
+    private final SecretStore secretStore;
+    private final EndpointCertificateProvider endpointCertificateProvider;
+    private final BooleanFlag useEndpointCertificateMaintainer;
+
+    public EndpointCertificateMaintainer(Controller controller, Duration interval) {
+        super(controller, interval, null, SystemName.allOf(Predicate.not(SystemName::isPublic)));
+        this.deploymentTrigger = controller.applications().deploymentTrigger();
+        this.clock = controller.clock();
+        this.secretStore = controller.secretStore();
+        this.curator = controller().curator();
+        this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider();
+        this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(controller().flagSource());
+    }
+
+    @Override
+    protected boolean maintain() {
+
+        if (!useEndpointCertificateMaintainer.value())
+            return true; // handled by EndpointCertificateManager for now
+
+        try {
+            // In order of importance
+            deployRefreshedCertificates();
+            updateRefreshedCertificates();
+            deleteUnusedCertificates();
+        } catch (Exception e) {
+            log.log(LogLevel.ERROR, "Exception caught while maintaining endpoint certificates", e);
+            return false;
+        }
+
+        return true;
+    }
+
+    private void updateRefreshedCertificates() {
+        curator.readAllEndpointCertificateMetadata().forEach(((applicationId, endpointCertificateMetadata) -> {
+            // Look for and use refreshed certificate
+            var latestAvailableVersion = latestVersionInSecretStore(endpointCertificateMetadata);
+            if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > endpointCertificateMetadata.version()) {
+                var refreshedCertificateMetadata = endpointCertificateMetadata
+                        .withVersion(latestAvailableVersion.getAsInt())
+                        .withLastRefreshed(clock.instant().getEpochSecond());
+                try (Lock lock = lock(applicationId)) {
+                    if (Optional.of(endpointCertificateMetadata).equals(curator.readEndpointCertificateMetadata(applicationId))) {
+                        curator.writeEndpointCertificateMetadata(applicationId, refreshedCertificateMetadata); // Certificate not validated here, but on deploy.
+                    }
+                }
+            }
+        }));
+    }
+
+    /**
+     * If it's been a week since the cert has been refreshed, re-trigger all prod deployment jobs.
+     */
+    private void deployRefreshedCertificates() {
+        var now = clock.instant();
+        curator.readAllEndpointCertificateMetadata().forEach((applicationId, endpointCertificateMetadata) ->
+                endpointCertificateMetadata.lastRefreshed().ifPresent(lastRefreshTime -> {
+                    Instant refreshTime = Instant.ofEpochSecond(lastRefreshTime);
+                    if (now.isAfter(refreshTime.plus(7, ChronoUnit.DAYS))) {
+
+                        controller().jobController().jobs(applicationId).forEach(job ->
+                                controller().jobController().jobStatus(new JobId(applicationId, JobType.fromJobName(job.jobName()))).lastTriggered().ifPresent(run -> {
+                                    if (run.start().isBefore(refreshTime) && job.isProduction() && job.isDeployment()) {
+                                        deploymentTrigger.reTrigger(applicationId, job);
+                                        log.info("Re-triggering deployment job " + job.jobName() + " for instance " +
+                                                applicationId.serializedForm() + " to roll out refreshed endpoint certificate");
+                                    }
+                                }));
+                    }
+                }));
+    }
+
+    private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) {
+        try {
+            var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName()));
+            var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName()));
+            return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max();
+        } catch (SecretNotFoundException s) {
+            return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version
+        }
+    }
+
+    private void deleteUnusedCertificates() {
+        var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS);
+        curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> {
+            var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested());
+            if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) {
+                try (Lock lock = lock(applicationId)) {
+                    if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) {
+                        log.log(Level.INFO, "Cert for app " + applicationId.serializedForm()
+                                + " has not been requested in a month and app has no deployments, deleting from provider and ZK");
+                        endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData);
+                        curator.deleteEndpointCertificateMetadata(applicationId);
+                    }
+                }
+            }
+        });
+    }
+
+    private Lock lock(ApplicationId applicationId) {
+        return curator.lock(TenantAndApplicationId.from(applicationId));
+    }
+
+    private boolean hasNoDeployments(ApplicationId applicationId) {
+        var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId))
+                .flatMap(app -> app.get(applicationId.instance()))
+                .map(Instance::deployments);
+
+        return deployments.isEmpty() || deployments.get().size() == 0;
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
index ba882ef7985..19f9542c679 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializer.java
@@ -7,7 +7,6 @@ import com.yahoo.slime.SlimeUtils;
 import com.yahoo.slime.Type;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 
-import java.util.List;
 import java.util.Optional;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
@@ -16,7 +15,7 @@ import java.util.stream.IntStream;
  * (de)serializes endpoint certificate metadata
  * <p>
  * A copy of package com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadata,
- * but will soon be extended as we need to store some more information in the controller.
+ * but with additional fields as we need to store some more information in the controller.
  *
  * @author andreer
  */
@@ -36,6 +35,8 @@ public class EndpointCertificateMetadataSerializer {
     private final static String requestIdField = "requestId";
     private final static String requestedDnsSansField = "requestedDnsSans";
     private final static String issuerField = "issuer";
+    private final static String expiryField = "expiry";
+    private final static String lastRefreshedField = "lastRefreshed";
 
     public static Slime toSlime(EndpointCertificateMetadata metadata) {
         Slime slime = new Slime();
@@ -44,13 +45,12 @@ public class EndpointCertificateMetadataSerializer {
         object.setString(certNameField, metadata.certName());
         object.setLong(versionField, metadata.version());
         object.setLong(lastRequestedField, metadata.lastRequested());
-
-        metadata.request_id().ifPresent(id -> object.setString(requestIdField, id));
-        metadata.requestedDnsSans().ifPresent(sans -> {
-            Cursor cursor = object.setArray(requestedDnsSansField);
-            sans.forEach(cursor::addString);
-        });
-        metadata.issuer().ifPresent(id -> object.setString(issuerField, id));
+        object.setString(requestIdField, metadata.request_id());
+        var cursor = object.setArray(requestedDnsSansField);
+        metadata.requestedDnsSans().forEach(cursor::addString);
+        object.setString(issuerField, metadata.issuer());
+        metadata.expiry().ifPresent(expiry -> object.setLong(expiryField, expiry));
+        metadata.lastRefreshed().ifPresent(refreshTime -> object.setLong(lastRefreshedField, refreshTime));
 
         return slime;
     }
@@ -58,32 +58,22 @@ public class EndpointCertificateMetadataSerializer {
     public static EndpointCertificateMetadata fromSlime(Inspector inspector) {
         if (inspector.type() != Type.OBJECT)
             throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
-        Optional<String> request_id = inspector.field(requestIdField).valid() ?
-                Optional.of(inspector.field(requestIdField).asString()) :
-                Optional.empty();
-
-        Optional<List<String>> requestedDnsSans = inspector.field(requestedDnsSansField).valid() ?
-                Optional.of(IntStream.range(0, inspector.field(requestedDnsSansField).entries())
-                        .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList())) :
-                Optional.empty();
-
-        Optional<String> issuer = inspector.field(issuerField).valid() ?
-                Optional.of(inspector.field(issuerField).asString()) :
-                Optional.empty();
-
-        long lastRequested = inspector.field(lastRequestedField).valid() ?
-                inspector.field(lastRequestedField).asLong() :
-                1597200000L; // Wed Aug 12 02:40:00 UTC 2020
-                // Not originally stored, so we default to when field was added
 
         return new EndpointCertificateMetadata(
                 inspector.field(keyNameField).asString(),
                 inspector.field(certNameField).asString(),
                 Math.toIntExact(inspector.field(versionField).asLong()),
-                lastRequested,
-                request_id,
-                requestedDnsSans,
-                issuer);
+                inspector.field(lastRequestedField).asLong(),
+                inspector.field(requestIdField).asString(),
+                IntStream.range(0, inspector.field(requestedDnsSansField).entries())
+                        .mapToObj(i -> inspector.field(requestedDnsSansField).entry(i).asString()).collect(Collectors.toList()),
+                inspector.field(issuerField).asString(),
+                inspector.field(expiryField).valid() ?
+                        Optional.of(inspector.field(expiryField).asLong()) :
+                        Optional.empty(),
+                inspector.field(lastRefreshedField).valid() ?
+                        Optional.of(inspector.field(lastRefreshedField).asLong()) :
+                        Optional.empty());
     }
 
     public static EndpointCertificateMetadata fromJsonString(String zkData) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
index 1e3771dedb0..4f2000b1902 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
@@ -114,7 +114,7 @@ public class EndpointCertificateManagerTest {
         assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
         assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
         assertEquals(0, endpointCertificateMetadata.get().version());
-        assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow());
+        assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans());
     }
 
     @Test
@@ -124,12 +124,18 @@ public class EndpointCertificateManagerTest {
         assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
         assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
         assertEquals(0, endpointCertificateMetadata.get().version());
-        assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow());
+        assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans());
     }
 
     @Test
     public void reuses_stored_certificate_metadata() {
-        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0));
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id",
+                List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud",
+                        "default.default.global.vespa.oath.cloud",
+                        "*.default.default.global.vespa.oath.cloud",
+                        "default.default.aws-us-east-1a.vespa.oath.cloud",
+                        "*.default.default.aws-us-east-1a.vespa.oath.cloud"),
+                "", Optional.empty(), Optional.empty()));
         secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7);
         secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7);
         Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
@@ -146,7 +152,13 @@ public class EndpointCertificateManagerTest {
         secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8);
         secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9);
         secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8);
-        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0));
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id",
+                List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud",
+                        "default.default.global.vespa.oath.cloud",
+                        "*.default.default.global.vespa.oath.cloud",
+                        "default.default.aws-us-east-1a.vespa.oath.cloud",
+                        "*.default.default.aws-us-east-1a.vespa.oath.cloud"),
+                "issuer", Optional.empty(), Optional.empty()));
         Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
         assertTrue(endpointCertificateMetadata.isPresent());
         assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
@@ -156,7 +168,7 @@ public class EndpointCertificateManagerTest {
 
     @Test
     public void reprovisions_certificate_when_necessary() {
-        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, Optional.of("uuid"), Optional.of(List.of()), Optional.empty()));
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty()));
         secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0);
         secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0);
         Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
@@ -169,7 +181,7 @@ public class EndpointCertificateManagerTest {
     public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() {
         ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId();
 
-        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, Optional.of("uuid"), Optional.of(expectedSans), Optional.of("mockCa")));
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", expectedSans, "mockCa", Optional.empty(), Optional.empty()));
         secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1);
         secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1);
 
@@ -180,7 +192,7 @@ public class EndpointCertificateManagerTest {
         assertTrue(endpointCertificateMetadata.isPresent());
         assertEquals(0, endpointCertificateMetadata.get().version());
         assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id()));
-        assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()));
+        assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans()));
     }
 
     @Test
@@ -202,6 +214,6 @@ public class EndpointCertificateManagerTest {
         assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
         assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
         assertEquals(0, endpointCertificateMetadata.get().version());
-        assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()));
+        assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans()));
     }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
new file mode 100644
index 00000000000..dbf102f23d7
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -0,0 +1,126 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.maintenance;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.util.List;
+import java.util.Optional;
+
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.productionUsWest1;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.stagingTest;
+import static com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType.systemTest;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author andreer
+ */
+public class EndpointCertificateMaintainerTest {
+
+    private final ControllerTester tester = new ControllerTester();
+    private final SecretStoreMock secretStore = (SecretStoreMock) tester.controller().secretStore();
+    private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1));
+    private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty());
+
+    @Before
+    public void setUp() throws Exception {
+        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.id(), true);
+    }
+
+    @Test
+    public void old_and_unused_cert_is_deleted() {
+        tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata);
+        assertTrue(maintainer.maintain());
+        assertTrue(tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()).isEmpty());
+    }
+
+    @Test
+    public void unused_but_recently_used_cert_is_not_deleted() {
+        EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond());
+        tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert);
+        assertTrue(maintainer.maintain());
+        assertEquals(Optional.of(recentlyRequestedCert), tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()));
+    }
+
+    @Test
+    public void refreshed_certificate_is_updated() {
+        EndpointCertificateMetadata recentlyRequestedCert = exampleMetadata.withLastRequested(tester.clock().instant().minusSeconds(3600).getEpochSecond());
+        tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), recentlyRequestedCert);
+
+        secretStore.setSecret(exampleMetadata.keyName(), "foo", 1);
+        secretStore.setSecret(exampleMetadata.certName(), "bar", 1);
+
+        assertTrue(maintainer.maintain());
+
+        var updatedCert = Optional.of(recentlyRequestedCert.withLastRefreshed(tester.clock().instant().getEpochSecond()).withVersion(1));
+
+        assertEquals(updatedCert, tester.curator().readEndpointCertificateMetadata(ApplicationId.defaultId()));
+    }
+
+    @Test
+    public void certificate_in_use_is_not_deleted() {
+        var appId = ApplicationId.from("tenant", "application", "default");
+
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+
+        var applicationPackage = new ApplicationPackageBuilder()
+                .region("us-west-1")
+                .build();
+
+        DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default");
+
+        deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1);
+
+
+        tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata);
+
+        assertTrue(maintainer.maintain());
+        assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed!
+    }
+
+    @Test
+    public void refreshed_certificate_is_deployed_after_one_week() {
+        var appId = ApplicationId.from("tenant", "application", "default");
+
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+
+        var applicationPackage = new ApplicationPackageBuilder()
+                .region("us-west-1")
+                .build();
+
+        DeploymentContext deploymentContext = deploymentTester.newDeploymentContext("tenant", "application", "default");
+
+        deploymentContext.submit(applicationPackage).runJob(systemTest).runJob(stagingTest).runJob(productionUsWest1);
+
+        tester.curator().writeEndpointCertificateMetadata(appId, exampleMetadata);
+
+        assertTrue(maintainer.maintain());
+        assertTrue(tester.curator().readEndpointCertificateMetadata(appId).isPresent()); // cert should not be deleted, the app is deployed!
+
+        tester.clock().advance(Duration.ofDays(3));
+
+        secretStore.setSecret(exampleMetadata.keyName(), "foo", 1);
+        secretStore.setSecret(exampleMetadata.certName(), "bar", 1);
+
+        maintainer.maintain();
+
+        tester.clock().advance(Duration.ofDays(8));
+
+        deploymentContext.assertNotRunning(productionUsWest1);
+
+        maintainer.maintain();
+
+        deploymentContext.assertRunning(productionUsWest1);
+    }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
index 9c6790f630b..00f5335bd82 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/EndpointCertificateMetadataSerializerTest.java
@@ -10,46 +10,39 @@ import static org.junit.Assert.*;
 
 public class EndpointCertificateMetadataSerializerTest {
 
-    private final EndpointCertificateMetadata sample =
-            new EndpointCertificateMetadata("keyName", "certName", 1, 0);
-    private final EndpointCertificateMetadata sampleWithRequestMetadata =
-            new EndpointCertificateMetadata("keyName", "certName", 1, 0, Optional.of("requestId"), Optional.of(List.of("SAN1", "SAN2")), Optional.of("issuer"));
+    private final EndpointCertificateMetadata sampleWithExpiryAndLastRefreshed =
+            new EndpointCertificateMetadata("keyName", "certName", 1, 0, "requestId", List.of("SAN1", "SAN2"), "issuer", java.util.Optional.of(1628000000L), Optional.of(1612000000L));
+
+    private final EndpointCertificateMetadata sampleWithoutExpiry =
+            new EndpointCertificateMetadata("keyName", "certName", 1, 0, "requestId", List.of("SAN1", "SAN2"), "issuer", Optional.empty(), Optional.empty());
 
     @Test
-    public void serialize() {
+    public void serializeWithExpiryAndLastRefreshed() {
         assertEquals(
-                "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0}",
-                EndpointCertificateMetadataSerializer.toSlime(sample).toString());
+                "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\",\"expiry\":1628000000,\"lastRefreshed\":1612000000}",
+                EndpointCertificateMetadataSerializer.toSlime(sampleWithExpiryAndLastRefreshed).toString());
     }
 
     @Test
-    public void serializeWithRequestMetadata() {
+    public void serializeWithoutExpiryAndLastRefreshed() {
         assertEquals(
                 "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}",
-                EndpointCertificateMetadataSerializer.toSlime(sampleWithRequestMetadata).toString());
+                EndpointCertificateMetadataSerializer.toSlime(sampleWithoutExpiry).toString());
     }
 
     @Test
-    public void deserializeFromJson() {
+    public void deserializeFromJsonWithExpiryAndLastRefreshed() {
         assertEquals(
-                sample,
+                sampleWithExpiryAndLastRefreshed,
                 EndpointCertificateMetadataSerializer.fromJsonString(
-                        "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0}"));
+                        "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\",\"expiry\":1628000000,\"lastRefreshed\":1612000000}"));
     }
 
     @Test
-    public void deserializeFromJsonWithRequestMetadata() {
+    public void deserializeFromJsonWithoutExpiryAndLastRefreshed() {
         assertEquals(
-                sampleWithRequestMetadata,
+                sampleWithoutExpiry,
                 EndpointCertificateMetadataSerializer.fromJsonString(
                         "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1,\"lastRequested\":0,\"requestId\":\"requestId\",\"requestedDnsSans\":[\"SAN1\",\"SAN2\"],\"issuer\":\"issuer\"}"));
     }
-
-    @Test
-    public void deserializeFromJsonWithDefaultLastRequested() {
-        assertEquals(
-                new EndpointCertificateMetadata("keyName", "certName", 1, 1597200000),
-                EndpointCertificateMetadataSerializer.fromJsonString(
-                        "{\"keyName\":\"keyName\",\"certName\":\"certName\",\"version\":1}"));
-    }
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 5af91e17bb7..6f67b0d8aa8 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -31,6 +31,9 @@
       "name": "DeploymentMetricsMaintainer"
     },
     {
+      "name": "EndpointCertificateMaintainer"
+    },
+    {
       "name": "HostSwitchUpdater"
     },
     {