Merge pull request #11842 from vespa-engine/andreer/endpoint-certificate-refactoring

accept and store json endpoint cert metadata on deploy

12 files changed, 375 insertions, 241 deletions

diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 52d47a9398b..4f847f86c0c 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -11,7 +11,7 @@ import com.yahoo.config.model.api.ContainerEndpoint;
 import com.yahoo.config.model.api.HostProvisioner;
 import com.yahoo.config.model.api.Model;
 import com.yahoo.config.model.api.ModelContext;
-import com.yahoo.config.model.api.TlsSecrets;
+import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.Zone;
@@ -130,7 +130,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean isBootstrap;
         private final boolean isFirstTimeDeployment;
         private final boolean useAdaptiveDispatch;
-        private final Optional<TlsSecrets> tlsSecrets;
+        private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
         private final double defaultTermwiseLimit;
         private final boolean useBucketSpaceMetric;
 
@@ -146,7 +146,7 @@ public class ModelContextImpl implements ModelContext {
                           boolean isBootstrap,
                           boolean isFirstTimeDeployment,
                           FlagSource flagSource,
-                          Optional<TlsSecrets> tlsSecrets) {
+                          Optional<EndpointCertificateSecrets> endpointCertificateSecrets) {
             this.applicationId = applicationId;
             this.multitenant = multitenantFromConfig || hostedVespa || Boolean.getBoolean("multitenant");
             this.configServerSpecs = configServerSpecs;
@@ -160,7 +160,7 @@ public class ModelContextImpl implements ModelContext {
             this.isFirstTimeDeployment = isFirstTimeDeployment;
             this.useAdaptiveDispatch = Flags.USE_ADAPTIVE_DISPATCH.bindTo(flagSource)
                     .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
-            this.tlsSecrets = tlsSecrets;
+            this.endpointCertificateSecrets = endpointCertificateSecrets;
             defaultTermwiseLimit = Flags.DEFAULT_TERM_WISE_LIMIT.bindTo(flagSource)
                     .with(FetchVector.Dimension.APPLICATION_ID, applicationId.serializedForm()).value();
             this.useBucketSpaceMetric = Flags.USE_BUCKET_SPACE_METRIC.bindTo(flagSource)
@@ -208,7 +208,7 @@ public class ModelContextImpl implements ModelContext {
         public boolean useAdaptiveDispatch() { return useAdaptiveDispatch; }
 
         @Override
-        public Optional<TlsSecrets> tlsSecrets() { return tlsSecrets; }
+        public Optional<EndpointCertificateSecrets> endpointCertificateSecrets() { return endpointCertificateSecrets; }
 
         @Override
         public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
index bc6419f230f..a2fc2bfd6a0 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/modelfactory/ActivatedModelsBuilder.java
@@ -27,8 +27,9 @@ import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.session.SessionZooKeeperClient;
 import com.yahoo.vespa.config.server.session.SilentDeployLogger;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever;
 import com.yahoo.vespa.config.server.tenant.TenantRepository;
-import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.flags.FlagSource;
 
@@ -135,7 +136,10 @@ public class ActivatedModelsBuilder extends ModelsBuilder<Application> {
                                                false, // We may be bootstrapping, but we only know and care during prepare
                                                false, // Always false, assume no one uses it when activating
                                                flagSource,
-                                               new TlsSecretsKeys(curator, TenantRepository.getTenantPath(tenant), secretStore).readTlsSecretsKeyFromZookeeper(applicationId));
+                                               new EndpointCertificateMetadataStore(curator, TenantRepository.getTenantPath(tenant))
+                                                       .readEndpointCertificateMetadata(applicationId)
+                                                       .flatMap(new EndpointCertificateRetriever(secretStore)::readEndpointCertificateSecrets));
+
     }
 
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
index ab3e0e863ce..1a41c1efd7a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/PrepareParams.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.config.server.session;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.model.api.ContainerEndpoint;
+import com.yahoo.config.model.api.EndpointCertificateMetadata;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.container.jdisc.HttpRequest;
@@ -11,6 +12,7 @@ import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.config.server.TimeoutBudget;
 import com.yahoo.vespa.config.server.http.SessionHandler;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointSerializer;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer;
 
 import java.time.Clock;
 import java.time.Duration;
@@ -32,6 +34,7 @@ public final class PrepareParams {
     static final String VESPA_VERSION_PARAM_NAME = "vespaVersion";
     static final String CONTAINER_ENDPOINTS_PARAM_NAME = "containerEndpoints";
     static final String TLS_SECRETS_KEY_NAME_PARAM_NAME = "tlsSecretsKeyName";
+    static final String ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME = "endpointCertificateMetadata";
 
     private final ApplicationId applicationId;
     private final TimeoutBudget timeoutBudget;
@@ -42,10 +45,12 @@ public final class PrepareParams {
     private final Optional<Version> vespaVersion;
     private final List<ContainerEndpoint> containerEndpoints;
     private final Optional<String> tlsSecretsKeyName;
+    private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
 
     private PrepareParams(ApplicationId applicationId, TimeoutBudget timeoutBudget, boolean ignoreValidationErrors,
                           boolean dryRun, boolean verbose, boolean isBootstrap, Optional<Version> vespaVersion,
-                          List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName) {
+                          List<ContainerEndpoint> containerEndpoints, Optional<String> tlsSecretsKeyName,
+                          Optional<EndpointCertificateMetadata> endpointCertificateMetadata) {
         this.timeoutBudget = timeoutBudget;
         this.applicationId = applicationId;
         this.ignoreValidationErrors = ignoreValidationErrors;
@@ -55,6 +60,7 @@ public final class PrepareParams {
         this.vespaVersion = vespaVersion;
         this.containerEndpoints = containerEndpoints;
         this.tlsSecretsKeyName = tlsSecretsKeyName;
+        this.endpointCertificateMetadata = endpointCertificateMetadata;
     }
 
     public static class Builder {
@@ -68,6 +74,7 @@ public final class PrepareParams {
         private Optional<Version> vespaVersion = Optional.empty();
         private List<ContainerEndpoint> containerEndpoints = List.of();
         private Optional<String> tlsSecretsKeyName = Optional.empty();
+        private Optional<EndpointCertificateMetadata> endpointCertificateMetadata = Optional.empty();
 
         public Builder() { }
 
@@ -128,9 +135,16 @@ public final class PrepareParams {
             return this;
         }
 
+        public Builder endpointCertificateMetadata(String serialized) {
+            if(serialized == null) return this;
+            Slime slime = SlimeUtils.jsonToSlime(serialized);
+            endpointCertificateMetadata = Optional.of(EndpointCertificateMetadataSerializer.fromSlime(slime.get()));
+            return this;
+        }
+
         public PrepareParams build() {
             return new PrepareParams(applicationId, timeoutBudget, ignoreValidationErrors, dryRun,
-                                     verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName);
+                                     verbose, isBootstrap, vespaVersion, containerEndpoints, tlsSecretsKeyName, endpointCertificateMetadata);
         }
 
     }
@@ -144,6 +158,7 @@ public final class PrepareParams {
                             .vespaVersion(request.getProperty(VESPA_VERSION_PARAM_NAME))
                             .containerEndpoints(request.getProperty(CONTAINER_ENDPOINTS_PARAM_NAME))
                             .tlsSecretsKeyName(request.getProperty(TLS_SECRETS_KEY_NAME_PARAM_NAME))
+                            .endpointCertificateMetadata(request.getProperty(ENDPOINT_CERTIFICATE_METADATA_PARAM_NAME))
                             .build();
     }
 
@@ -200,4 +215,8 @@ public final class PrepareParams {
     public Optional<String> tlsSecretsKeyName() {
         return tlsSecretsKeyName;
     }
+
+    public Optional<EndpointCertificateMetadata> endpointCertificateMetadata() {
+        return endpointCertificateMetadata;
+    }
 }
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
index 171eab35507..0115876ded9 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/session/SessionPreparer.java
@@ -12,8 +12,9 @@ import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.application.api.FileRegistry;
 import com.yahoo.config.model.api.ConfigDefinitionRepo;
 import com.yahoo.config.model.api.ContainerEndpoint;
+import com.yahoo.config.model.api.EndpointCertificateMetadata;
 import com.yahoo.config.model.api.ModelContext;
-import com.yahoo.config.model.api.TlsSecrets;
+import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.provision.AllocatedHosts;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
@@ -33,7 +34,9 @@ import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
 import com.yahoo.vespa.config.server.modelfactory.PreparedModelsBuilder;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
-import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataSerializer;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever;
 import com.yahoo.vespa.curator.Curator;
 import com.yahoo.vespa.flags.FlagSource;
 import org.xml.sax.SAXException;
@@ -113,7 +116,7 @@ public class SessionPreparer {
             preparation.makeResult(allocatedHosts);
             if ( ! params.isDryRun()) {
                 preparation.writeStateZK();
-                preparation.writeTlsZK();
+                preparation.writeEndpointCertificateMetadataZK();
                 preparation.writeContainerEndpointsZK();
                 preparation.distribute();
             }
@@ -142,8 +145,10 @@ public class SessionPreparer {
         final ContainerEndpointsCache containerEndpoints;
         final Set<ContainerEndpoint> endpointsSet;
         final ModelContext.Properties properties;
-        private final TlsSecretsKeys tlsSecretsKeys;
-        private final Optional<TlsSecrets> tlsSecrets;
+        private final EndpointCertificateMetadataStore endpointCertificateMetadataStore;
+        private final EndpointCertificateRetriever endpointCertificateRetriever;
+        private final Optional<EndpointCertificateMetadata> endpointCertificateMetadata;
+        private final Optional<EndpointCertificateSecrets> endpointCertificateSecrets;
 
         private ApplicationPackage applicationPackage;
         private List<PreparedModelsBuilder.PreparedModelResult> modelResultList;
@@ -162,8 +167,16 @@ public class SessionPreparer {
             this.applicationId = params.getApplicationId();
             this.vespaVersion = params.vespaVersion().orElse(Vtag.currentVersion);
             this.containerEndpoints = new ContainerEndpointsCache(tenantPath, curator);
-            this.tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore);
-            this.tlsSecrets = tlsSecretsKeys.getTlsSecrets(params.tlsSecretsKeyName(), applicationId);
+            this.endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath);
+            this.endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore);
+
+            this.endpointCertificateMetadata = params.endpointCertificateMetadata()
+                    .or(() -> params.tlsSecretsKeyName().map(EndpointCertificateMetadataSerializer::fromString));
+
+            endpointCertificateSecrets = endpointCertificateMetadata
+                    .or(() -> endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId))
+                    .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
+
             this.endpointsSet = getEndpoints(params.containerEndpoints());
 
             this.properties = new ModelContextImpl.Properties(params.getApplicationId(),
@@ -178,7 +191,7 @@ public class SessionPreparer {
                                                               params.isBootstrap(),
                                                               ! currentActiveApplicationSet.isPresent(),
                                                               context.getFlagSource(),
-                                                              tlsSecrets);
+                                                              endpointCertificateSecrets);
             this.preparedModelsBuilder = new PreparedModelsBuilder(modelFactoryRegistry,
                                                                    permanentApplicationPackage,
                                                                    configDefinitionRepo,
@@ -233,9 +246,10 @@ public class SessionPreparer {
             checkTimeout("write state to zookeeper");
         }
 
-        void writeTlsZK() {
-            tlsSecretsKeys.writeTlsSecretsKeyToZooKeeper(applicationId, params.tlsSecretsKeyName().orElse(null));
-            checkTimeout("write tlsSecretsKey to zookeeper");
+        void writeEndpointCertificateMetadataZK() {
+            endpointCertificateMetadata.ifPresent(metadata ->
+                    endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, metadata));
+            checkTimeout("write endpoint certificate metadata to zookeeper");
         }
 
         void writeContainerEndpointsZK() {
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java
new file mode 100644
index 00000000000..6d092aaa18b
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataSerializer.java
@@ -0,0 +1,55 @@
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.config.model.api.EndpointCertificateMetadata;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Inspector;
+import com.yahoo.slime.Slime;
+
+/**
+ * (de)serializes endpoint certificate metadata
+ *
+ * @author andreer
+ */
+public class EndpointCertificateMetadataSerializer {
+
+    // WARNING: Since there are multiple servers in a ZooKeeper cluster and they upgrade one by one
+    //          (and rewrite all nodes on startup), changes to the serialized format must be made
+    //          such that what is serialized on version N+1 can be read by version N:
+    //          - ADDING FIELDS: Always ok
+    //          - REMOVING FIELDS: Stop reading the field first. Stop writing it on a later version.
+    //          - CHANGING THE FORMAT OF A FIELD: Don't do it bro.
+
+    private final static String keyNameField = "keyName";
+    private final static String certNameField = "certName";
+    private final static String versionField = "version";
+
+    public static void toSlime(EndpointCertificateMetadata metadata, Cursor object) {
+        object.setString(keyNameField, metadata.keyName());
+        object.setString(certNameField, metadata.certName());
+        object.setLong(versionField, metadata.version());
+    }
+
+    public static EndpointCertificateMetadata fromSlime(Inspector inspector) {
+        switch (inspector.type()) {
+            case STRING: // TODO: Remove once all are transmitted and stored as JSON
+                return new EndpointCertificateMetadata(
+                        inspector.asString() + "-key",
+                        inspector.asString() + "-cert",
+                        0
+                );
+            case OBJECT:
+                return new EndpointCertificateMetadata(
+                        inspector.field(keyNameField).asString(),
+                        inspector.field(certNameField).asString(),
+                        Math.toIntExact(inspector.field(versionField).asLong())
+                );
+
+            default:
+                throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!");
+        }
+    }
+
+    public static EndpointCertificateMetadata fromString(String tlsSecretsKeys) {
+        return fromSlime(new Slime().setString(tlsSecretsKeys));
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
new file mode 100644
index 00000000000..6500449e557
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStore.java
@@ -0,0 +1,65 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.config.model.api.EndpointCertificateMetadata;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.path.Path;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.curator.Curator;
+import com.yahoo.vespa.curator.transaction.CuratorOperations;
+import com.yahoo.vespa.curator.transaction.CuratorTransaction;
+
+import java.util.Optional;
+
+/**
+ * Stores the endpoint certificate metadata for an application.
+ * This metadata is then used to retrieve the actual secrets from {@link EndpointCertificateRetriever}.
+ *
+ * @author andreer
+ */
+public class EndpointCertificateMetadataStore {
+
+    private final Path path;
+    private final Curator curator;
+
+    public EndpointCertificateMetadataStore(Curator curator, Path tenantPath) {
+        this.curator = curator;
+        this.path = tenantPath.append("tlsSecretsKeys/");
+    }
+
+    /** Reads the endpoint certificate metadata from ZooKeeper, if it exists */
+    public Optional<EndpointCertificateMetadata> readEndpointCertificateMetadata(ApplicationId application) {
+        try {
+            Optional<byte[]> data = curator.getData(endpointCertificateMetadataPathOf(application));
+            if (data.isEmpty() || data.get().length == 0) return Optional.empty();
+            Slime slime = SlimeUtils.jsonToSlime(data.get());
+            EndpointCertificateMetadata endpointCertificateMetadata = EndpointCertificateMetadataSerializer.fromSlime(slime.get());
+            return Optional.of(endpointCertificateMetadata);
+        } catch (Exception e) {
+            throw new RuntimeException("Error reading TLS secret key of " + application, e);
+        }
+    }
+
+    /** Writes the endpoint certificate metadata to ZooKeeper */
+    public void writeEndpointCertificateMetadata(ApplicationId application, EndpointCertificateMetadata endpointCertificateMetadata) {
+        try {
+            Slime slime = new Slime();
+            EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata, slime.setObject());
+            curator.set(endpointCertificateMetadataPathOf(application), SlimeUtils.toJsonBytes(slime));
+        } catch (Exception e) {
+            throw new RuntimeException("Could not write TLS secret key of " + application, e);
+        }
+    }
+
+    /** Returns a transaction which deletes these tls secrets key if they exist */
+    public CuratorTransaction delete(ApplicationId application) {
+        if (!curator.exists(endpointCertificateMetadataPathOf(application))) return CuratorTransaction.empty(curator);
+        return CuratorTransaction.from(CuratorOperations.delete(endpointCertificateMetadataPathOf(application).getAbsolute()), curator);
+    }
+
+    /** Returns the path storing the tls secrets key for an application */
+    private Path endpointCertificateMetadataPathOf(ApplicationId application) {
+        return path.append(application.serializedForm());
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java
new file mode 100644
index 00000000000..5f40e5e1411
--- /dev/null
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateRetriever.java
@@ -0,0 +1,56 @@
+// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.config.model.api.EndpointCertificateMetadata;
+import com.yahoo.config.model.api.EndpointCertificateSecrets;
+import com.yahoo.container.jdisc.secretstore.SecretStore;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.X509CertificateUtils;
+
+import java.security.PrivateKey;
+import java.security.PublicKey;
+import java.security.cert.X509Certificate;
+import java.util.Optional;
+
+/**
+ * Used to retrieve actual endpoint certificate/key from secret store.
+ *
+ * @author andreer
+ */
+public class EndpointCertificateRetriever {
+
+    private final SecretStore secretStore;
+
+    public EndpointCertificateRetriever(SecretStore secretStore) {
+        this.secretStore = secretStore;
+    }
+
+    public Optional<EndpointCertificateSecrets> readEndpointCertificateSecrets(EndpointCertificateMetadata metadata) {
+        return Optional.of(readFromSecretStore(metadata));
+    }
+
+    private EndpointCertificateSecrets readFromSecretStore(EndpointCertificateMetadata endpointCertificateMetadata) {
+        try {
+            String cert = secretStore.getSecret(endpointCertificateMetadata.certName(), endpointCertificateMetadata.version());
+            String key = secretStore.getSecret(endpointCertificateMetadata.keyName(), endpointCertificateMetadata.version());
+
+            verifyKeyMatchesCertificate(endpointCertificateMetadata, cert, key);
+
+            return new EndpointCertificateSecrets(cert, key);
+        } catch (RuntimeException e) {
+            // Assume not ready yet
+            return EndpointCertificateSecrets.MISSING;
+        }
+    }
+
+    private void verifyKeyMatchesCertificate(EndpointCertificateMetadata endpointCertificateMetadata, String cert, String key) {
+        X509Certificate x509Certificate = X509CertificateUtils.fromPem(cert);
+
+        PrivateKey privateKey = KeyUtils.fromPemEncodedPrivateKey(key);
+        PublicKey publicKey = x509Certificate.getPublicKey();
+
+        if(!X509CertificateUtils.privateKeyMatchesPublicKey(privateKey, publicKey)) {
+            throw new IllegalArgumentException("Failed to retrieve endpoint secrets: Certificate and key data do not match for " + endpointCertificateMetadata);
+        }
+    }
+}
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java b/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java
deleted file mode 100644
index da6fc490da9..00000000000
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeys.java
+++ /dev/null
@@ -1,136 +0,0 @@
-// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.tenant;
-
-import com.yahoo.config.model.api.TlsSecrets;
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.container.jdisc.secretstore.SecretStore;
-import com.yahoo.path.Path;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
-import com.yahoo.vespa.curator.Curator;
-import com.yahoo.vespa.curator.transaction.CuratorOperations;
-import com.yahoo.vespa.curator.transaction.CuratorTransaction;
-
-import java.util.Optional;
-
-/**
- * TLS Secret keys for applications (used to retrieve actual certificate/key from secret store). Persisted in ZooKeeper.
- *
- * @author andreer
- */
-public class TlsSecretsKeys {
-
-    private final Path path;
-    private final SecretStore secretStore;
-    private final Curator curator;
-
-    public TlsSecretsKeys(Curator curator, Path tenantPath, SecretStore secretStore) {
-        this.curator = curator;
-        this.path = tenantPath.append("tlsSecretsKeys/");
-        this.secretStore = secretStore;
-    }
-
-    public Optional<TlsSecrets> readTlsSecretsKeyFromZookeeper(ApplicationId application) {
-        try {
-            Optional<byte[]> data = curator.getData(tlsSecretsKeyOf(application));
-            if (data.isEmpty() || data.get().length == 0) return Optional.empty();
-
-            Slime slime = SlimeUtils.jsonToSlime(data.get());
-            final var inspector = slime.get();
-
-            switch (inspector.type()) {
-                case STRING: // TODO: Remove once all are stored as JSON
-                    return readFromSecretStore(Optional.ofNullable(inspector.asString()));
-                case OBJECT:
-                    var tlsSecretsInfo = new TlsSecretsMetadata();
-                    tlsSecretsInfo.certName = inspector.field("certName").asString();
-                    tlsSecretsInfo.keyName = inspector.field("keyName").asString();
-                    tlsSecretsInfo.version = Math.toIntExact(inspector.field("version").asLong());
-                    return Optional.of(readFromSecretStore(tlsSecretsInfo));
-                default:
-                    throw new IllegalArgumentException("Unknown format encountered for TLS secrets metadata!");
-            }
-        } catch (Exception e) {
-            throw new RuntimeException("Error reading TLS secret key of " + application, e);
-        }
-    }
-
-    public void writeTlsSecretsKeyToZooKeeper(ApplicationId application, String tlsSecretsKey) {
-        if (tlsSecretsKey == null) return;
-        writeTlsSecretsAsString(application, tlsSecretsKey);
-    }
-
-    private void writeTlsSecretsAsString(ApplicationId application, String tlsSecretsKey) {
-        try {
-            Slime slime = new Slime();
-            slime.setString(tlsSecretsKey);
-            curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime));
-        } catch (Exception e) {
-            throw new RuntimeException("Could not write TLS secret key of " + application, e);
-        }
-    }
-
-    void writeTlsSecretsMetadata(ApplicationId application, TlsSecretsMetadata tlsSecretsMetadata) {
-        try {
-            Slime slime = new Slime();
-            Cursor cursor = slime.setObject();
-            cursor.setString(TlsSecretsMetadata.certNameField, tlsSecretsMetadata.certName);
-            cursor.setString(TlsSecretsMetadata.keyNameField, tlsSecretsMetadata.keyName);
-            cursor.setLong(TlsSecretsMetadata.versionField, tlsSecretsMetadata.version);
-            curator.set(tlsSecretsKeyOf(application), SlimeUtils.toJsonBytes(slime));
-        } catch (Exception e) {
-            throw new RuntimeException("Could not write TLS secret key of " + application, e);
-        }
-    }
-
-    public Optional<TlsSecrets> getTlsSecrets(Optional<String> secretKeyname, ApplicationId applicationId) {
-        if (secretKeyname == null || secretKeyname.isEmpty()) {
-            return readTlsSecretsKeyFromZookeeper(applicationId);
-        }
-        return readFromSecretStore(secretKeyname);
-    }
-
-    private Optional<TlsSecrets> readFromSecretStore(Optional<String> secretKeyname) {
-        if (secretKeyname.isEmpty()) return Optional.empty();
-        try {
-            String cert = secretStore.getSecret(secretKeyname.get() + "-cert");
-            String key = secretStore.getSecret(secretKeyname.get() + "-key");
-            return Optional.of(new TlsSecrets(cert, key));
-        } catch (RuntimeException e) {
-            // Assume not ready yet
-            return Optional.of(TlsSecrets.MISSING);
-        }
-    }
-
-    private TlsSecrets readFromSecretStore(TlsSecretsMetadata tlsSecretsMetadata) {
-        try {
-            String cert = secretStore.getSecret(tlsSecretsMetadata.certName, tlsSecretsMetadata.version);
-            String key = secretStore.getSecret(tlsSecretsMetadata.keyName, tlsSecretsMetadata.version);
-            return new TlsSecrets(cert, key);
-        } catch (RuntimeException e) {
-            // Assume not ready yet
-            return TlsSecrets.MISSING;
-        }
-    }
-
-    /** Returns a transaction which deletes these tls secrets key if they exist */
-    public CuratorTransaction delete(ApplicationId application) {
-        if (!curator.exists(tlsSecretsKeyOf(application))) return CuratorTransaction.empty(curator);
-        return CuratorTransaction.from(CuratorOperations.delete(tlsSecretsKeyOf(application).getAbsolute()), curator);
-    }
-
-    /** Returns the path storing the tls secrets key for an application */
-    private Path tlsSecretsKeyOf(ApplicationId application) {
-        return path.append(application.serializedForm());
-    }
-
-    static class TlsSecretsMetadata {
-        final static String keyNameField = "keyName";
-        final static String certNameField = "certName";
-        final static String versionField = "version";
-        String keyName;
-        String certName;
-        int version;
-    }
-}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java
index 8a77b53875e..12f48778144 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/MockSecretStore.java
@@ -7,22 +7,26 @@ import java.util.HashMap;
 import java.util.Map;
 
 public class MockSecretStore implements SecretStore {
-    Map<String, String> secrets = new HashMap<>();
+    Map<String, Map<Integer, String>> secrets = new HashMap<>();
 
     @Override
     public String getSecret(String key) {
         if(secrets.containsKey(key))
-            return secrets.get(key);
+            return secrets.get(key).get(0);
         throw new RuntimeException("Key not found: " + key);
     }
 
     @Override
     public String getSecret(String key, int version) {
-        return getSecret(key);
+        return secrets.get(key).get(version);
+    }
+
+    public void put(String key, int version, String value) {
+        secrets.computeIfAbsent(key, k -> new HashMap<>()).put(version, value);
     }
 
     public void put(String key, String value) {
-        secrets.put(key, value);
+        put(key, 0, value);
     }
 
     public void remove(String key) {
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
index a099db5ebe8..40115170b69 100644
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/session/SessionPreparerTest.java
@@ -4,7 +4,7 @@ package com.yahoo.vespa.config.server.session;
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeployLogger;
 import com.yahoo.config.model.api.ContainerEndpoint;
-import com.yahoo.config.model.api.TlsSecrets;
+import com.yahoo.config.model.api.EndpointCertificateSecrets;
 import com.yahoo.config.model.application.provider.BaseDeployLogger;
 import com.yahoo.config.model.application.provider.FilesApplicationPackage;
 import com.yahoo.config.provision.ApplicationId;
@@ -22,6 +22,11 @@ import com.yahoo.config.provision.exception.LoadBalancerServiceException;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
 import com.yahoo.path.Path;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.slime.Slime;
 import com.yahoo.transaction.NestedTransaction;
 import com.yahoo.vespa.config.server.MockReloadHandler;
@@ -37,7 +42,8 @@ import com.yahoo.vespa.config.server.model.TestModelFactory;
 import com.yahoo.vespa.config.server.modelfactory.ModelFactoryRegistry;
 import com.yahoo.vespa.config.server.provision.HostProvisionerProvider;
 import com.yahoo.vespa.config.server.tenant.ContainerEndpointsCache;
-import com.yahoo.vespa.config.server.tenant.TlsSecretsKeys;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateMetadataStore;
+import com.yahoo.vespa.config.server.tenant.EndpointCertificateRetriever;
 import com.yahoo.vespa.config.server.zookeeper.ConfigCurator;
 import com.yahoo.vespa.curator.mock.MockCurator;
 import com.yahoo.vespa.flags.InMemoryFlagSource;
@@ -46,9 +52,14 @@ import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TemporaryFolder;
 
+import javax.security.auth.x500.X500Principal;
 import java.io.File;
 import java.io.IOException;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
 import java.time.Instant;
+import java.time.temporal.ChronoUnit;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
@@ -73,6 +84,9 @@ public class SessionPreparerTest {
     private static final File invalidTestApp = new File("src/test/apps/illegalApp");
     private static final Version version123 = new Version(1, 2, 3);
     private static final Version version321 = new Version(3, 2, 1);
+    private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+    private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"),
+            Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build();
 
     private final InMemoryFlagSource flagSource = new InMemoryFlagSource();
     private MockCurator curator;
@@ -231,15 +245,37 @@ public class SessionPreparerTest {
         var tlskey = "vespa.tlskeys.tenant1--app1";
         var applicationId = applicationId("test");
         var params = new PrepareParams.Builder().applicationId(applicationId).tlsSecretsKeyName(tlskey).build();
-        secretStore.put(tlskey+"-cert", "CERT");
-        secretStore.put(tlskey+"-key", "KEY");
+
+        secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate));
+        secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate()));
+
         prepare(new File("src/test/resources/deploy/hosted-app"), params);
 
         // Read from zk and verify cert and key are available
-        Optional<TlsSecrets> tlsSecrets = new TlsSecretsKeys(curator, tenantPath, secretStore).readTlsSecretsKeyFromZookeeper(applicationId);
-        assertTrue(tlsSecrets.isPresent());
-        assertEquals("KEY", tlsSecrets.get().key());
-        assertEquals("CERT", tlsSecrets.get().certificate());
+        Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath)
+                .readEndpointCertificateMetadata(applicationId)
+                .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p));
+        assertTrue(endpointCertificateSecrets.isPresent());
+        assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY"));
+        assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE"));
+    }
+
+    @Test
+    public void require_that_endpoint_certificate_metadata_is_written() throws IOException {
+        var applicationId = applicationId("test");
+        var params = new PrepareParams.Builder().applicationId(applicationId).endpointCertificateMetadata("{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 7}").build();
+        secretStore.put("vespa.tlskeys.tenant1--app1-cert", 7, X509CertificateUtils.toPem(certificate));
+        secretStore.put("vespa.tlskeys.tenant1--app1-key", 7, KeyUtils.toPem(keyPair.getPrivate()));
+        prepare(new File("src/test/resources/deploy/hosted-app"), params);
+
+        // Read from zk and verify cert and key are available
+        Optional<EndpointCertificateSecrets> endpointCertificateSecrets = new EndpointCertificateMetadataStore(curator, tenantPath)
+                .readEndpointCertificateMetadata(applicationId)
+                .flatMap(p -> new EndpointCertificateRetriever(secretStore).readEndpointCertificateSecrets(p));
+
+        assertTrue(endpointCertificateSecrets.isPresent());
+        assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY"));
+        assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE"));
     }
 
     @Test(expected = CertificateNotReadyException.class)
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java
new file mode 100644
index 00000000000..d71eab25ce3
--- /dev/null
+++ b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/EndpointCertificateMetadataStoreTest.java
@@ -0,0 +1,90 @@
+// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.config.server.tenant;
+
+import com.yahoo.config.model.api.EndpointCertificateMetadata;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ApplicationName;
+import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.TenantName;
+import com.yahoo.path.Path;
+import com.yahoo.security.KeyAlgorithm;
+import com.yahoo.security.KeyUtils;
+import com.yahoo.security.SignatureAlgorithm;
+import com.yahoo.security.X509CertificateBuilder;
+import com.yahoo.security.X509CertificateUtils;
+import com.yahoo.vespa.config.server.MockSecretStore;
+import com.yahoo.vespa.curator.mock.MockCurator;
+import org.junit.Before;
+import org.junit.Test;
+
+import javax.security.auth.x500.X500Principal;
+import java.math.BigInteger;
+import java.security.KeyPair;
+import java.security.cert.X509Certificate;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+public class EndpointCertificateMetadataStoreTest {
+
+    private static final Path tenantPath = Path.createRoot();
+    private static final Path endpointCertificateMetadataPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default");
+    private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(),
+            ApplicationName.from("test"), InstanceName.defaultName());
+
+    private MockCurator curator;
+    private MockSecretStore secretStore = new MockSecretStore();
+    private EndpointCertificateMetadataStore endpointCertificateMetadataStore;
+    private EndpointCertificateRetriever endpointCertificateRetriever;
+    private KeyPair keyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 256);
+    private X509Certificate certificate = X509CertificateBuilder.fromKeypair(keyPair, new X500Principal("CN=subject"),
+            Instant.now(), Instant.now().plus(1, ChronoUnit.DAYS), SignatureAlgorithm.SHA512_WITH_ECDSA, BigInteger.valueOf(12345)).build();
+
+    @Before
+    public void setUp() {
+        curator = new MockCurator();
+        endpointCertificateMetadataStore = new EndpointCertificateMetadataStore(curator, tenantPath);
+        endpointCertificateRetriever = new EndpointCertificateRetriever(secretStore);
+
+        secretStore.put("vespa.tlskeys.tenant1--app1-cert", X509CertificateUtils.toPem(certificate));
+        secretStore.put("vespa.tlskeys.tenant1--app1-key", KeyUtils.toPem(keyPair.getPrivate()));
+    }
+
+    @Test
+    public void reads_string_format() {
+        curator.set(endpointCertificateMetadataPath, ("\"vespa.tlskeys.tenant1--app1\"").getBytes());
+
+        // Read from zk and verify cert and key are available
+        var endpointCertificateSecrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)
+                .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
+        assertTrue(endpointCertificateSecrets.isPresent());
+        assertTrue(endpointCertificateSecrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY"));
+        assertTrue(endpointCertificateSecrets.get().certificate().startsWith("-----BEGIN CERTIFICATE"));
+    }
+
+    @Test
+    public void reads_object_format() {
+        curator.set(endpointCertificateMetadataPath,
+                "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}"
+                        .getBytes());
+
+        // Read from zk and verify cert and key are available
+        var secrets = endpointCertificateMetadataStore.readEndpointCertificateMetadata(applicationId)
+                .flatMap(endpointCertificateRetriever::readEndpointCertificateSecrets);
+        assertTrue(secrets.isPresent());
+        assertTrue(secrets.get().key().startsWith("-----BEGIN EC PRIVATE KEY"));
+        assertTrue(secrets.get().certificate().startsWith("-----BEGIN CERTIFICATE"));
+    }
+
+    @Test
+    public void can_write_object_format() {
+        var endpointCertificateMetadata = new EndpointCertificateMetadata("key-name", "cert-name", 1);
+
+        endpointCertificateMetadataStore.writeEndpointCertificateMetadata(applicationId, endpointCertificateMetadata);
+
+        assertEquals("{\"keyName\":\"key-name\",\"certName\":\"cert-name\",\"version\":1}",
+                new String(curator.getData(endpointCertificateMetadataPath).orElseThrow()));
+    }
+}
diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java
deleted file mode 100644
index c71c7b8e040..00000000000
--- a/configserver/src/test/java/com/yahoo/vespa/config/server/tenant/TlsSecretsKeysTest.java
+++ /dev/null
@@ -1,73 +0,0 @@
-// Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
-package com.yahoo.vespa.config.server.tenant;
-
-import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.config.provision.ApplicationName;
-import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.TenantName;
-import com.yahoo.path.Path;
-import com.yahoo.vespa.config.server.MockSecretStore;
-import com.yahoo.vespa.curator.mock.MockCurator;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-public class TlsSecretsKeysTest {
-
-    private static final Path tenantPath = Path.createRoot();
-    private static final Path tlsSecretsKeysPath = Path.createRoot().append("tlsSecretsKeys").append("default:test:default");
-    private static final String tlskey = "vespa.tlskeys.tenant1--app1";
-    private static final ApplicationId applicationId = ApplicationId.from(TenantName.defaultName(),
-            ApplicationName.from("test"), InstanceName.defaultName());
-
-    private MockCurator curator;
-    private MockSecretStore secretStore = new MockSecretStore();
-    private TlsSecretsKeys tlsSecretsKeys;
-
-    @Before
-    public void setUp() {
-        curator = new MockCurator();
-        tlsSecretsKeys = new TlsSecretsKeys(curator, tenantPath, secretStore);
-        secretStore.put(tlskey + "-cert", "CERT");
-        secretStore.put(tlskey + "-key", "KEY");
-    }
-
-    @Test
-    public void reads_string_format() {
-        curator.set(tlsSecretsKeysPath, ('"' + tlskey + '"').getBytes());
-
-        // Read from zk and verify cert and key are available
-        var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId);
-        assertTrue(tlsSecrets.isPresent());
-        assertEquals("KEY", tlsSecrets.get().key());
-        assertEquals("CERT", tlsSecrets.get().certificate());
-    }
-
-    @Test
-    public void reads_object_format() {
-        curator.set(tlsSecretsKeysPath,
-                "{\"keyName\": \"vespa.tlskeys.tenant1--app1-key\", \"certName\":\"vespa.tlskeys.tenant1--app1-cert\", \"version\": 0}"
-                        .getBytes());
-
-        // Read from zk and verify cert and key are available
-        var tlsSecrets = tlsSecretsKeys.readTlsSecretsKeyFromZookeeper(applicationId);
-        assertTrue(tlsSecrets.isPresent());
-        assertEquals("KEY", tlsSecrets.get().key());
-        assertEquals("CERT", tlsSecrets.get().certificate());
-    }
-
-    @Test
-    public void can_write_object_format() {
-        var tlsSecretsMetadata = new TlsSecretsKeys.TlsSecretsMetadata();
-        tlsSecretsMetadata.certName = "cert-name";
-        tlsSecretsMetadata.keyName = "key-name";
-        tlsSecretsMetadata.version = 1;
-
-        tlsSecretsKeys.writeTlsSecretsMetadata(applicationId, tlsSecretsMetadata);
-
-        assertEquals("{\"certName\":\"cert-name\",\"keyName\":\"key-name\",\"version\":1}",
-                new String(curator.getData(tlsSecretsKeysPath).get()));
-    }
-}