Assign random id to existing certificates

2 files changed, 222 insertions, 0 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
index dea5d048fc5..dd87dff653d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -3,18 +3,26 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.google.common.collect.Sets;
 import com.yahoo.component.annotation.Inject;
+import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.transaction.Mutex;
 import com.yahoo.transaction.NestedTransaction;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.PermanentFlags;
+import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateDetails;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateRequest;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.GeneratedEndpoint;
 import com.yahoo.vespa.hosted.controller.certificate.UnassignedCertificate;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.secrets.EndpointSecretManager;
@@ -34,9 +42,11 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.OptionalInt;
+import java.util.Set;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 /**
  * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates.
@@ -56,6 +66,9 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
     private final EndpointSecretManager endpointSecretManager;
     private final EndpointCertificateProvider endpointCertificateProvider;
     final Comparator<EligibleJob> oldestFirst = Comparator.comparing(e -> e.deployment.at());
+    final BooleanFlag assignRandomizedId;
+    private final StringFlag endpointCertificateAlgo;
+    private final BooleanFlag useAlternateCertProvider;
 
     @Inject
     public EndpointCertificateMaintainer(Controller controller, Duration interval) {
@@ -66,6 +79,9 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
         this.endpointSecretManager = controller.serviceRegistry().secretManager();
         this.curator = controller().curator();
         this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider();
+        this.assignRandomizedId = Flags.ASSIGN_RANDOMIZED_ID.bindTo(controller.flagSource());
+        this.useAlternateCertProvider = PermanentFlags.USE_ALTERNATIVE_ENDPOINT_CERTIFICATE_PROVIDER.bindTo(controller.flagSource());
+        this.endpointCertificateAlgo = PermanentFlags.ENDPOINT_CERTIFICATE_ALGORITHM.bindTo(controller.flagSource());
     }
 
     @Override
@@ -76,6 +92,7 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
             updateRefreshedCertificates();
             deleteUnusedCertificates();
             deleteOrReportUnmanagedCertificates();
+            assignRandomizedIds();
         } catch (Exception e) {
             log.log(Level.SEVERE, "Exception caught while maintaining endpoint certificates", e);
             return 1.0;
@@ -252,6 +269,110 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
         }
     }
 
+    private void assignRandomizedIds() {
+        List<AssignedCertificate> assignedCertificates = curator.readAssignedCertificates();
+            /*
+                only assign randomized id if:
+                * instance is present
+                * randomized id is not already assigned
+                * feature flag is enabled
+            */
+        assignedCertificates.stream()
+                .filter(c -> c.instance().isPresent())
+                .filter(c -> c.certificate().randomizedId().isEmpty())
+                .filter(c -> assignRandomizedId.with(FetchVector.Dimension.APPLICATION_ID, c.application().instance(c.instance().get()).serializedForm()).value())
+                .forEach(c -> assignRandomizedId(c.application(), c.instance().get()));
+    }
+
+    /*
+        Assign randomized id according to these rules:
+        * Instance is not mentioned in the deployment spec for this application
+            -> assume this is a manual deployment. Assign a randomized id to the certificate, save using instance only
+        * Instance is mentioned in deployment spec:
+            -> If there is a random endpoint assigned to tenant:application -> use this also for the "instance" certificate
+            -> Otherwise assign a random endpoint and write to the application and the instance.
+     */
+    private void assignRandomizedId(TenantAndApplicationId tenantAndApplicationId, InstanceName instanceName) {
+        Optional<AssignedCertificate> assignedCertificate = curator.readAssignedCertificate(tenantAndApplicationId, Optional.of(instanceName));
+        if (assignedCertificate.isEmpty()) {
+            log.log(Level.INFO, "Assigned certificate missing for " + tenantAndApplicationId.instance(instanceName).toFullString() + " when assigning randomized id");
+        }
+        // Verify that the assigned certificate still does not have randomized id assigned
+        if (assignedCertificate.get().certificate().randomizedId().isPresent()) return;
+
+        controller().applications().lockApplicationOrThrow(tenantAndApplicationId, application -> {
+            DeploymentSpec deploymentSpec = application.get().deploymentSpec();
+            if (deploymentSpec.instance(instanceName).isPresent()) {
+                Optional<AssignedCertificate> applicationLevelAssignedCertificate  = curator.readAssignedCertificate(tenantAndApplicationId, Optional.empty());
+                assignApplicationRandomId(assignedCertificate.get(), applicationLevelAssignedCertificate);
+            } else {
+                assignInstanceRandomId(assignedCertificate.get());
+            }
+        });
+    }
+
+    private void assignApplicationRandomId(AssignedCertificate instanceLevelAssignedCertificate, Optional<AssignedCertificate> applicationLevelAssignedCertificate) {
+        TenantAndApplicationId tenantAndApplicationId = instanceLevelAssignedCertificate.application();
+        if (applicationLevelAssignedCertificate.isPresent()) {
+            applicationLevelAssignedCertificate.get().certificate().randomizedId().orElseThrow(() -> new IllegalArgumentException("Application certificate already assigned to " + tenantAndApplicationId.toString() + ", but random id is missing"));
+            // Application level assigned certificate with randomized id already exists. Copy randomized id to instance level certificate and request with random names.
+            EndpointCertificate withRandomNames = requestRandomNames(tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), applicationLevelAssignedCertificate.get().certificate().randomizedId().get(), Optional.of(instanceLevelAssignedCertificate.certificate()));
+            AssignedCertificate assignedCertWithRandomNames = instanceLevelAssignedCertificate.with(withRandomNames);
+            curator.writeAssignedCertificate(assignedCertWithRandomNames);
+        } else {
+            // No application level certificate exists, generate new assigned certificate with the randomized id based names only, then request same names also for instance level cert
+            String randomId = generateRandomId();
+            EndpointCertificate applicationLevelEndpointCert = requestRandomNames(tenantAndApplicationId, Optional.empty(), randomId, Optional.empty());
+            AssignedCertificate applicationLevelCert = new AssignedCertificate(tenantAndApplicationId, Optional.empty(), applicationLevelEndpointCert);
+
+            EndpointCertificate instanceLevelEndpointCert = requestRandomNames(tenantAndApplicationId, instanceLevelAssignedCertificate.instance(), randomId, Optional.of(instanceLevelAssignedCertificate.certificate()));
+            instanceLevelAssignedCertificate = instanceLevelAssignedCertificate.with(instanceLevelEndpointCert);
+
+            // Save both in transaction
+            try (NestedTransaction transaction = new NestedTransaction()) {
+                curator.writeAssignedCertificate(instanceLevelAssignedCertificate, transaction);
+                curator.writeAssignedCertificate(applicationLevelCert, transaction);
+                transaction.commit();
+            }
+        }
+    }
+
+    private void assignInstanceRandomId(AssignedCertificate assignedCertificate) {
+        String randomId = generateRandomId();
+        EndpointCertificate withRandomNames = requestRandomNames(assignedCertificate.application(), assignedCertificate.instance(), randomId, Optional.of(assignedCertificate.certificate()));
+        AssignedCertificate assignedCertWithRandomNames = assignedCertificate.with(withRandomNames);
+        curator.writeAssignedCertificate(assignedCertWithRandomNames);
+    }
+
+    private EndpointCertificate requestRandomNames(TenantAndApplicationId tenantAndApplicationId, Optional<InstanceName> instanceName, String randomId, Optional<EndpointCertificate> previousRequest) {
+        String dnsSuffix = Endpoint.dnsSuffix(controller().system());
+        List<String> newSanDnsEntries = List.of(
+                "*.%s.z%s".formatted(randomId, dnsSuffix),
+                "*.%s.g%s".formatted(randomId, dnsSuffix),
+                "*.%s.a%s".formatted(randomId, dnsSuffix));
+        List<String> existingSanDnsEntries = previousRequest.map(EndpointCertificate::requestedDnsSans).orElse(List.of());
+        List<String> requestNames = Stream.concat(existingSanDnsEntries.stream(), newSanDnsEntries.stream()).toList();
+        String key = instanceName.map(tenantAndApplicationId::instance).map(ApplicationId::toFullString).orElseGet(tenantAndApplicationId::toString);
+        return endpointCertificateProvider.requestCaSignedCertificate(
+                        key,
+                        requestNames,
+                        previousRequest,
+                        endpointCertificateAlgo.value(),
+                        useAlternateCertProvider.value())
+                .withRandomizedId(randomId);
+    }
+
+    private String generateRandomId() {
+        List<String> unassignedIds = curator.readUnassignedCertificates().stream().map(UnassignedCertificate::id).toList();
+        List<String> assignedIds = curator.readAssignedCertificates().stream().map(AssignedCertificate::certificate).map(EndpointCertificate::randomizedId).filter(Optional::isPresent).map(Optional::get).toList();
+        Set<String> allIds = Stream.concat(unassignedIds.stream(), assignedIds.stream()).collect(Collectors.toSet());
+        String randomId;
+        do {
+            randomId = GeneratedEndpoint.createPart(controller().random(true));
+        } while (allIds.contains(randomId));
+        return randomId;
+    }
+
     private static String asString(TenantAndApplicationId application, Optional<InstanceName> instanceName) {
         return application.toString() + instanceName.map(name -> "." + name.value()).orElse("");
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
index 422df76f1fb..31e290c6448 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -6,6 +6,8 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.jdisc.test.MockMetric;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificate;
@@ -32,6 +34,8 @@ import java.util.Optional;
 import java.util.OptionalDouble;
 import java.util.stream.Stream;
 
+import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.devUsEast1;
+import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.perfUsEast3;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest;
@@ -174,6 +178,103 @@ public class EndpointCertificateMaintainerTest {
         assertNotEquals(List.of(), endpointCertificateProvider.listCertificates());
     }
 
+    @Test
+    void certificates_are_not_assigned_random_id_when_flag_disabled() {
+        var app = ApplicationId.from("tenant", "app", "default");
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+        deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty());
+        assertEquals(1, tester.curator().readAssignedCertificates().size());
+
+        maintainer.maintain();
+        assertEquals(1, tester.curator().readAssignedCertificates().size());
+    }
+
+    @Test
+    void production_deployment_certificates_are_assigned_random_id() {
+        var app = ApplicationId.from("tenant", "app", "default");
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+        deployToAssignCert(deploymentTester, app, List.of(systemTest, stagingTest, productionUsWest1), Optional.empty());
+        assertEquals(1, tester.curator().readAssignedCertificates().size());
+
+        ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true);
+        maintainer.maintain();
+        assertEquals(2, tester.curator().readAssignedCertificates().size());
+
+        // Verify random id is same for application and instance certificates
+        Optional<AssignedCertificate> applicationCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.empty());
+        assertTrue(applicationCertificate.isPresent());
+        Optional<AssignedCertificate> instanceCertificate = tester.curator().readAssignedCertificate(TenantAndApplicationId.from(app), Optional.of(app.instance()));
+        assertTrue(instanceCertificate.isPresent());
+        assertEquals(instanceCertificate.get().certificate().randomizedId(), applicationCertificate.get().certificate().randomizedId());
+
+        // Verify the 3 wildcard random names are same in all certs
+        List<String> appWildcardSans = applicationCertificate.get().certificate().requestedDnsSans();
+        List<String> instanceSans = instanceCertificate.get().certificate().requestedDnsSans();
+        assertEquals(3, appWildcardSans.size());
+        List<String> wildcards = instanceSans.stream().filter(appWildcardSans::contains).toList();
+        assertEquals(3, wildcards.size());
+    }
+
+    @Test
+    void existing_application_randomid_is_copied_to_new_instance_deployments() {
+        var instance1 = ApplicationId.from("tenant", "prod", "instance1");
+        var instance2 = ApplicationId.from("tenant", "prod", "instance2");
+
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+        deployToAssignCert(deploymentTester, instance1, List.of(systemTest, stagingTest,productionUsWest1),Optional.of("instance1"));
+        assertEquals(1, tester.curator().readAssignedCertificates().size());
+        ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true);
+        maintainer.maintain();
+
+        String randomId = tester.curator().readAssignedCertificate(instance1).get().certificate().randomizedId().get();
+
+        deployToAssignCert(deploymentTester, instance2, List.of(productionUsWest1), Optional.of("instance1,instance2"));
+        maintainer.maintain();
+        assertEquals(3, tester.curator().readAssignedCertificates().size());
+
+        assertEquals(randomId, tester.curator().readAssignedCertificate(instance1).get().certificate().randomizedId().get());
+    }
+
+    @Test
+    void dev_certificates_are_not_assigned_application_level_certificate() {
+        var devApp = ApplicationId.from("tenant", "devonly", "foo");
+        DeploymentTester deploymentTester = new DeploymentTester(tester);
+        deployToAssignCert(deploymentTester, devApp, List.of(devUsEast1), Optional.empty());
+        assertEquals(1, tester.curator().readAssignedCertificates().size());
+        ((InMemoryFlagSource)deploymentTester.controller().flagSource()).withBooleanFlag(Flags.ASSIGN_RANDOMIZED_ID.id(), true);
+        List<String> originalRequestedSans = tester.curator().readAssignedCertificate(devApp).get().certificate().requestedDnsSans();
+        maintainer.maintain();
+        assertEquals(1, tester.curator().readAssignedCertificates().size());
+
+        // Verify certificate is assigned random id and 3 new names
+        Optional<AssignedCertificate> assignedCertificate = tester.curator().readAssignedCertificate(devApp);
+        assertTrue(assignedCertificate.get().certificate().randomizedId().isPresent());
+        List<String> newRequestedSans = assignedCertificate.get().certificate().requestedDnsSans();
+        List<String> randomizedNames = newRequestedSans.stream().filter(san -> !originalRequestedSans.contains(san)).toList();
+        assertEquals(3, randomizedNames.size());
+    }
+
+    private void deployToAssignCert(DeploymentTester tester, ApplicationId applicationId, List<JobType> jobTypes, Optional<String> instances) {
+        var applicationPackageBuilder = new ApplicationPackageBuilder()
+                .region("us-west-1");
+        instances.map(applicationPackageBuilder::instances);
+        var applicationPackage = applicationPackageBuilder.build();
+
+        List<JobType> manualJobs = jobTypes.stream().filter(jt -> jt.environment().isManuallyDeployed()).toList();
+        List<JobType> jobs = jobTypes.stream().filter(jt -> ! jt.environment().isManuallyDeployed()).toList();
+
+        DeploymentContext deploymentContext = tester.newDeploymentContext(applicationId);
+        deploymentContext.submit(applicationPackage);
+        manualJobs.forEach(job -> deploymentContext.runJob(job, applicationPackage));
+        jobs.forEach(deploymentContext::runJob);
+
+    }
+    EndpointCertificate certificate(List<String> sans) {
+        return new EndpointCertificate("keyName", "certName", 0, 0, "root-request-uuid", Optional.of("leaf-request-uuid"), List.of(), "issuer", Optional.empty(), Optional.empty(), Optional.empty());
+    }
+
+
+
     private static AssignedCertificate assignedCertificate(ApplicationId instance, EndpointCertificate certificate) {
         return new AssignedCertificate(TenantAndApplicationId.from(instance), Optional.of(instance.instance()), certificate);
     }