andreer/endpoint certificate maintainer (#16099)

* remove support for old formats and introduce EndpointCertificateMaintainer

* record certificate refresh time, run maintainer every 12 hours

* retrigger prod deployments if refreshed certificate not deployed after one week

* only re-trigger production jobs

* unit test EndpointCertificateMaintainer

* take application lock to avoid concurrent modifications when managing endpoint certs

* only trigger deployment jobs

Co-authored-by: Jon Marius Venstad <jonmv@users.noreply.github.com>

6 files changed, 20 insertions, 83 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index 139081fde00..78071cbf89e 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -37,7 +37,6 @@ public final class PrepareParams {
     static final String VERBOSE_PARAM_NAME = "verbose";
     static final String VESPA_VERSION_PARAM_NAME = "vespaVersion";
     static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints";
-    static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName";
     static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata";
     static final String DOCKER_IMAGE_REPOSITORY = "dockerImageRepository";
     static final String ATHENZ_DOMAIN = "athenzDomain";
@@ -55,7 +54,6 @@ public final class PrepareParams {
     private final boolean force;
     private final Optional<Version> vespaVersion;
     private final List<ContainerEndpoint> containerEndpoints;
-    private final Optional<String> tlsSecretsKeyName;
     private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
     private final Optional<DockerImage> dockerImageRepository;
     private final Optional<AthenzDomain> athenzDomain;
@@ -64,7 +62,7 @@ public final class PrepareParams {
 
     private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors,
                           boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion,
-                          List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName,
+                          List<ContainerEndpoint> containerEndpoints,
                           Optional<EndpointCertificateMetadata> endpointCertificateMetadata,
                           Optional<DockerImage> dockerImageRepository, Optional<AthenzDomain> athenzDomain,
                           Optional<ApplicationRoles> applicationRoles, Optional<Quota> quota, boolean force) {
@@ -76,7 +74,6 @@ public final class PrepareParams {
         this.isBootstrap = isBootstrap;
         this.vespaVersion = vespaVersion;
         this.containerEndpoints = containerEndpoints;
-        this.tlsSecretsKeyName = tlsSecretsKeyName;
         this.endpointCertificateMetadata = endpointCertificateMetadata;
         this.dockerImageRepository = dockerImageRepository;
         this.athenzDomain = athenzDomain;
@@ -96,7 +93,6 @@ public final class PrepareParams {
         private TimeoutBudget timeoutBudget = new TimeoutBudget(Clock.systemUTC(), Duration.ofSeconds(60));
         private Optional<Version> vespaVersion = Optional.empty();
         private List<ContainerEndpoint> containerEndpoints = null;
-        private Optional<String> tlsSecretsKeyName = Optional.empty();
         private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty();
         private Optional<DockerImage> dockerImageRepository = Optional.empty();
         private Optional<AthenzDomain> athenzDomain = Optional.empty();
@@ -156,12 +152,6 @@ public final class PrepareParams {
             return this;
         }
 
-        public Builder tlsSecretsKeyName(String tlsSecretsKeyName) {
-            this.tlsSecretsKeyName = Optional.ofNullable(tlsSecretsKeyName)
-                                           .filter(s -> ! s.isEmpty());
-            return this;
-        }
-
         public Builder endpointCertificateMetadata(String serialized) {
             this.endpointCertificateMetadata = (serialized == null)
                     ? Optional.empty()
@@ -210,7 +200,7 @@ public final class PrepareParams {
 
         public PrepareParams build() {
             return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun,
-                                     verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName,
+                                     verbose, isBootstrap, vespaVersion, containerEndpoints,
                                      endpointCertificateMetadata, dockerImageRepository, athenzDomain,
                                      applicationRoles, quota, force);
         }
@@ -224,7 +214,6 @@ public final class PrepareParams {
                             .applicationId(createApplicationId(request, tenant))
                             .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME))
                             .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME))
-                            .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME))
                             .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME))
                             .dockerImageRepository(request.getProperty(DOCKER_IMAGE_REPOSITORY))
                             .athenzDomain(request.getProperty(ATHENZ_DOMAIN))
@@ -284,10 +273,6 @@ public final class PrepareParams {
         return timeoutBudget;
     }
 
-    public Optional<String> tlsSecretsKeyName() {
-        return tlsSecretsKeyName;
-    }
-
     public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() {
         return endpointCertificateMetadata;
     }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index c244274c49f..b29259e22d4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -181,8 +181,7 @@ public class SessionPreparer {
             this.containerEndpointsCache = new ContainerEndpointsCache(tenantPath, curator);
             this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath);
             EndpointCertificateRetriever endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore);
-            this.endpointCertificateMetadata = params.endpointCertificateMetadata()
-                    .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString));
+            this.endpointCertificateMetadata = params.endpointCertificateMetadata();
             Optional<EndpointCertificateSecrets> endpointCertificateSecrets = endpointCertificateMetadata
                     .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId))
                     .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java
index f3240a62133..13c88ece6d7 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java
@@ -3,7 +3,7 @@ package com.yahoo.vespa.config.server.tenant;
 import com.yahoo.config.model.api.EndpointCertificateMetadata;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
-import com.yahoo.slime.Slime;
+import com.yahoo.slime.Type;
 
 /**
  * (de)serializes endpoint certificate metadata
@@ -30,26 +30,13 @@ public class EndpointCertificateMetadataSerializer {
     }
 
     public static EndpointCertificateMetadata fromSlime(Inspector inspector) {
-        switch (inspector.type()) {
-            case STRING: // TODO: Remove once all are transmitted and stored as JSON
-                return new EndpointCertificateMetadata(
-                        inspector.asString() + "-key",
-                        inspector.asString() + "-cert",
-                        0
-                );
-            case OBJECT:
-                return new EndpointCertificateMetadata(
-                        inspector.field(keyNameField).asString(),
-                        inspector.field(certNameField).asString(),
-                        Math.toIntExact(inspector.field(versionField).asLong())
-                );
-
-            default:
-                throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
+        if (inspector.type() == Type.OBJECT) {
+            return new EndpointCertificateMetadata(
+                    inspector.field(keyNameField).asString(),
+                    inspector.field(certNameField).asString(),
+                    Math.toIntExact(inspector.field(versionField).asLong())
+            );
         }
-    }
-
-    public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) {
-        return fromSlime(new Slime().setString(tlsSecretsKeys));
+        throw new IllegalArgumentException("Unknown format encountered for endpoint certificate metadata!");
     }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
index 8e51ac424f9..415f0a41441 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
@@ -37,7 +37,7 @@ public class EndpointCertificateMetadataStore {
             EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get());
             return Optional.of(endpointCertificateMetadata);
         } catch (Exception e) {
-            throw new RuntimeException("Error reading TLS secret key of " + application, e);
+            throw new RuntimeException("Error reading endpoint certificate metadata for " + application, e);
         }
     }
 
@@ -48,17 +48,17 @@ public class EndpointCertificateMetadataStore {
             EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject());
             curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime));
         } catch (Exception e) {
-            throw new RuntimeException("Could not write TLS secret key of " + application, e);
+            throw new RuntimeException("Could not write endpoint certificate metadata for " + application, e);
         }
     }
 
-    /** Returns a transaction which deletes these tls secrets key if they exist */
+    /** Returns a transaction which deletes endpoint certificate metadata if it exists */
     public CuratorTransaction delete(ApplicationId application) {
         if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator);
         return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator);
     }
 
-    /** Returns the path storing the tls secrets key for an application */
+    /** Returns the path storing the endpoint certificate metadata for an application */
     private Path endpointCertificateMetadataPathOf(ApplicationId application) {
         return path.append(application.serializedForm());
     }
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index c719efd7645..5a49090031c 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -289,27 +289,6 @@ public class SessionPreparerTest {
     }
 
     @Test
-    public void require_that_tlssecretkey_is_written() throws IOException {
-        var tlskey = "vespa.tlskeys.tenant1--app1";
-        var applicationId = applicationId("test");
-        var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build();
-
-        secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate));
-        secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate()));
-
-        prepare(new File("src/test/resources/deploy/hosted-app"), params);
-
-        // Read from zk and verify cert and key are available
-        Path tenantPath = TenantRepository.getTenantPath(applicationId.tenant());
-        Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath)
-                .readEndpointCertificateMetadata(applicationId)
-                .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p));
-        assertTrue(endpointCertificateSecrets.isPresent());
-        assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY"));
-        assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE"));
-    }
-
-    @Test
     public void require_that_endpoint_certificate_metadata_is_written() throws IOException {
         var applicationId = applicationId("test");
         var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build();
@@ -329,19 +308,18 @@ public class SessionPreparerTest {
     }
 
     @Test(expected = CertificateNotReadyException.class)
-    public void require_that_tlssecretkey_is_missing_when_not_in_secretstore() throws IOException {
-        var tlskey = "vespa.tlskeys.tenant1--app1";
+    public void endpoint_certificate_is_missing_when_not_in_secretstore() throws IOException {
         var applicationId = applicationId("test");
-        var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build();
+        var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build();
         prepare(new File("src/test/resources/deploy/hosted-app"), params);
     }
 
     @Test(expected = CertificateNotReadyException.class)
-    public void require_that_tlssecretkey_is_missing_when_certificate_not_in_secretstore() throws IOException {
+    public void endpoint_certificate_is_missing_when_certificate_not_in_secretstore() throws IOException {
         var tlskey = "vespa.tlskeys.tenant1--app1";
         var applicationId = applicationId("test");
-        var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build();
-        secretStore.put(tlskey+"-key", "KEY");
+        var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build();
+        secretStore.put(tlskey+"-key", 7, "KEY");
         prepare(new File("src/test/resources/deploy/hosted-app"), params);
     }
 
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java
index 829e47bdb42..1a4c7c9669e 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java
@@ -53,18 +53,6 @@ public class EndpointCertificateMetadataStoreTest {
     }
 
     @Test
-    public void reads_string_format() {
-        curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes());
-
-        // Read from zk and verify cert and key are available
-        var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)
-                .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
-        assertTrue(endpointCertificateSecrets.isPresent());
-        assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY"));
-        assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE"));
-    }
-
-    @Test
     public void reads_object_format() {
         curator.set(endpointCertificateMetadataPath,
                 "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}"