Merge pull request #12134 from vespa-engine/andreer/certificate-metadata-maintainer

backfill certificate metadata from provider

5 files changed, 105 insertions, 2 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
index aa0ac5f8296..8e81400f3c8 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateMock.java
@@ -29,4 +29,9 @@ public class EndpointCertificateMock implements EndpointCertificateProvider {
         return new EndpointCertificateMetadata(endpointCertificatePrefix + "-key", endpointCertificatePrefix + "-cert", 0);
     }
 
+    @Override
+    public List<EndpointCertificateMetadata> listCertificates() {
+        return Collections.emptyList();
+    }
+
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
index 147ada51816..97d2bdb3343 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/certificates/EndpointCertificateProvider.java
@@ -14,4 +14,5 @@ public interface EndpointCertificateProvider  {
 
     EndpointCertificateMetadata requestCaSignedCertificate(ApplicationId applicationId, List<String> dnsNames);
 
+    List<EndpointCertificateMetadata> listCertificates();
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
index c90d5886777..b702afe1647 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.flags.BooleanFlag;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
@@ -27,11 +28,15 @@ import java.time.Clock;
 import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.Set;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -52,6 +57,7 @@ public class EndpointCertificateManager {
     private final EndpointCertificateProvider endpointCertificateProvider;
     private final Clock clock;
     private final BooleanFlag useRefreshedEndpointCertificate;
+    private final StringFlag endpointCertificateBackfill;
 
     public EndpointCertificateManager(ZoneRegistry zoneRegistry,
                                       CuratorDb curator,
@@ -64,6 +70,8 @@ public class EndpointCertificateManager {
         this.endpointCertificateProvider = endpointCertificateProvider;
         this.clock = clock;
         this.useRefreshedEndpointCertificate = Flags.USE_REFRESHED_ENDPOINT_CERTIFICATE.bindTo(flagSource);
+        this.endpointCertificateBackfill = Flags.ENDPOINT_CERTIFICATE_BACKFILL.bindTo(flagSource);
+        Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(this::backfillCertificateMetadata, 1, 10, TimeUnit.MINUTES);
     }
 
     public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) {
@@ -97,6 +105,76 @@ public class EndpointCertificateManager {
         return Optional.of(endpointCertificateMetadata);
     }
 
+    enum BackfillMode {
+        DISABLE,
+        DRYRUN,
+        ENABLE;
+
+        BackfillMode fromString(String backfillMode) {
+            switch (backfillMode) {
+                case "disable":
+                    return DISABLE;
+                case "dryrun":
+                    return DRYRUN;
+                case "enable":
+                    return ENABLE;
+                default:
+                    throw new RuntimeException("Uknown backfill mode!");
+            }
+        }
+    }
+
+    private void backfillCertificateMetadata() {
+        BackfillMode mode = BackfillMode.valueOf(endpointCertificateBackfill.value());
+        if (mode == BackfillMode.DISABLE) return;
+
+        List<EndpointCertificateMetadata> allProviderCertificateMetadata = endpointCertificateProvider.listCertificates();
+        Map<String, EndpointCertificateMetadata> sanToEndpointCertificate = new HashMap<>();
+
+        allProviderCertificateMetadata.forEach((providerMetadata -> {
+            if (providerMetadata.request_id().isEmpty())
+                throw new RuntimeException("Backfill failed - provider metadata missing request_id");
+            if (providerMetadata.requestedDnsSans().isEmpty())
+                throw new RuntimeException("Backfill failed - provider metadata missing DNS SANs for " + providerMetadata.request_id().get());
+            providerMetadata.requestedDnsSans().get().forEach(san -> {
+                        var previous = sanToEndpointCertificate.put(san, providerMetadata);
+                        if (previous != null)
+                            throw new RuntimeException("Backfill failed - Overlapping SANs in certificates " +
+                                    providerMetadata.request_id() + " and " + previous.request_id());
+                    }
+            );
+        }));
+
+        Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = curator.readAllEndpointCertificateMetadata();
+
+        allEndpointCertificateMetadata.forEach((applicationId, storedMetaData) -> {
+            if (storedMetaData.requestedDnsSans().isPresent() && storedMetaData.request_id().isPresent())
+                return;
+
+            var hashedCn = Endpoint.createHashedCn(applicationId, zoneRegistry.system()); // use as join key
+            EndpointCertificateMetadata providerMetadata = sanToEndpointCertificate.get(hashedCn);
+
+            if(providerMetadata == null) {
+                log.log(LogLevel.INFO, "No matching certificate provider metadata found for application " + applicationId.serializedForm());
+                return;
+            }
+
+            EndpointCertificateMetadata backfilledMetadata =
+                    new EndpointCertificateMetadata(
+                            storedMetaData.keyName(),
+                            storedMetaData.certName(),
+                            storedMetaData.version(),
+                            providerMetadata.request_id(),
+                            providerMetadata.requestedDnsSans());
+
+            if (mode == BackfillMode.DRYRUN) {
+                log.log(LogLevel.INFO, "Would update stored metadata " + storedMetaData + " with data from provider: " + backfilledMetadata);
+            } else if (mode == BackfillMode.ENABLE) {
+                curator.writeEndpointCertificateMetadata(applicationId, backfilledMetadata);
+            }
+        });
+    }
+
     private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) {
         var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName()));
         var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName()));
@@ -140,7 +218,7 @@ public class EndpointCertificateManager {
                     .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME))
                     .map(SubjectAlternativeName::getValue).collect(Collectors.toSet());
 
-            if(Sets.intersection(subjectAlternativeNames, Set.copyOf(dnsNamesOf(instance.id(), List.of(zone)))).isEmpty()) {
+            if (Sets.intersection(subjectAlternativeNames, Set.copyOf(dnsNamesOf(instance.id(), List.of(zone)))).isEmpty()) {
                 return logWarning(warningPrefix, "No overlap between SANs in certificate and expected SANs");
             }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index 5cd5bb59a9b..b15daa19d6c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -40,7 +40,9 @@ import java.nio.ByteBuffer;
 import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 import java.util.NavigableMap;
@@ -512,7 +514,7 @@ public class CuratorDb {
                                                      .orElse(new ZoneRoutingPolicy(zone, GlobalRouting.DEFAULT_STATUS));
     }
 
-    // -------------- Application web certificates ----------------------------
+    // -------------- Application endpoint certificates ----------------------------
 
     public void writeEndpointCertificateMetadata(ApplicationId applicationId, EndpointCertificateMetadata endpointCertificateMetadata) {
         curator.set(endpointCertificatePath(applicationId), asJson(EndpointCertificateMetadataSerializer.toSlime(endpointCertificateMetadata)));
@@ -523,6 +525,17 @@ public class CuratorDb {
         return zkData.map(EndpointCertificateMetadataSerializer::fromJsonOrTlsSecretsKeysString);
     }
 
+    public Map<ApplicationId, EndpointCertificateMetadata> readAllEndpointCertificateMetadata() {
+        Map<ApplicationId, EndpointCertificateMetadata> allEndpointCertificateMetadata = new HashMap<>();
+        Iterator<String> zkNodes = endpointCertificateRoot.iterator();
+        while(zkNodes.hasNext()) {
+            ApplicationId applicationId = ApplicationId.fromSerializedForm(zkNodes.next());
+            Optional<EndpointCertificateMetadata> endpointCertificateMetadata = readEndpointCertificateMetadata(applicationId);
+            allEndpointCertificateMetadata.put(applicationId, endpointCertificateMetadata.orElseThrow());
+        }
+        return allEndpointCertificateMetadata;
+    }
+
     // -------------- Paths ---------------------------------------------------
 
     private Path lockPath(TenantName tenant) {
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index c1767799f9e..babe804e6e4 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -218,6 +218,12 @@ public class Flags {
             "Takes effect on the next deployment of the application",
             APPLICATION_ID);
 
+    public static final UnboundStringFlag ENDPOINT_CERTIFICATE_BACKFILL = defineStringFlag(
+            "endpoint-certificate-backfill", "disable",
+            "Whether the endpoint certificate maintainer should backfill missing certificate data from cameo",
+            "Takes effect on next scheduled run of maintainer - set to \"disable\", \"dryrun\" or \"enable\""
+    );
+
     public static final UnboundBooleanFlag USE_NEW_ATHENZ_FILTER = defineFeatureFlag(
             "use-new-athenz-filter", false,
             "Use new Athenz filter that supports access-tokens",