Merge pull request #13374 from vespa-engine/andreer/minimize-requested-SANs

reduce number of SANs in endpoint certificates

4 files changed, 170 insertions, 43 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index 3592b37b7fe..d644cf21638 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -330,7 +330,7 @@ public class ApplicationController {
                     &&   run.testerCertificate().isPresent())
                     applicationPackage = applicationPackage.withTrustedCertificate(run.testerCertificate().get());
 
-                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone);
+                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, zone, applicationPackage.deploymentSpec().instance(instance.name()));
 
                 endpoints = controller.routing().registerEndpointsInDns(application.get(), job.application().instance(), zone);
 
@@ -413,7 +413,8 @@ public class ApplicationController {
                     validateRun(application.get().require(instance), zone, platformVersion, applicationVersion);
                 }
 
-                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(application.get().require(instance), zone);
+                endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(
+                        application.get().require(instance), zone, applicationPackage.deploymentSpec().instance(instance));
 
                 endpoints = controller.routing().registerEndpointsInDns(application.get(), instance, zone);
             } // Release application lock while doing the deployment, which is a lengthy task.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
index 7e0fec4ba66..513709aefc8 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
@@ -3,15 +3,20 @@ package com.yahoo.vespa.hosted.controller.certificate;
 import com.google.common.collect.Sets;
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
+import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
+
+import java.util.LinkedHashSet;
 import java.util.logging.Level;
+
 import com.yahoo.security.SubjectAlternativeName;
 import com.yahoo.security.X509CertificateUtils;
 import com.yahoo.vespa.flags.BooleanFlag;
@@ -91,16 +96,16 @@ public class EndpointCertificateManager {
         }, 1, 10, TimeUnit.MINUTES);
     }
 
-    public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone) {
+    public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) {
         var t0 = Instant.now();
-        Optional<EndpointCertificateMetadata> metadata = getOrProvision(instance, zone);
+        Optional<EndpointCertificateMetadata> metadata = getOrProvision(instance, zone, instanceSpec);
         Duration duration = Duration.between(t0, Instant.now());
         if (duration.toSeconds() > 30) log.log(Level.INFO, String.format("Getting endpoint certificate metadata for %s took %d seconds!", instance.id().serializedForm(), duration.toSeconds()));
         return metadata;
     }
 
     @NotNull
-    private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone) {
+    private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) {
         boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value();
         if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting)
             return Optional.empty();
@@ -108,7 +113,7 @@ public class EndpointCertificateManager {
         final var currentCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id());
 
         if (currentCertificateMetadata.isEmpty()) {
-            var provisionedCertificateMetadata = provisionEndpointCertificate(instance, Optional.empty());
+            var provisionedCertificateMetadata = provisionEndpointCertificate(instance, Optional.empty(), zone, instanceSpec);
             // We do not verify the certificate if one has never existed before - because we do not want to
             // wait for it to be available before we deploy. This allows the config server to start
             // provisioning nodes ASAP, and the risk is small for a new deployment.
@@ -118,9 +123,9 @@ public class EndpointCertificateManager {
 
         // Reprovision certificate if it is missing SANs for the zone we are deploying to
         var sansInCertificate = currentCertificateMetadata.get().requestedDnsSans();
-        var requiredSansForZone = dnsNamesOf(instance.id(), List.of(zone));
+        var requiredSansForZone = dnsNamesOf(instance.id(), zone);
         if (sansInCertificate.isPresent() && !sansInCertificate.get().containsAll(requiredSansForZone)) {
-            var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata);
+            var reprovisionedCertificateMetadata = provisionEndpointCertificate(instance, currentCertificateMetadata, zone, instanceSpec);
             curator.writeEndpointCertificateMetadata(instance.id(), reprovisionedCertificateMetadata);
             // Verification is unlikely to succeed in this case, as certificate must be available first - controller will retry
             validateEndpointCertificate(reprovisionedCertificateMetadata, instance, zone);
@@ -206,9 +211,41 @@ public class EndpointCertificateManager {
         }
     }
 
-    private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata) {
-        List<ZoneId> zones = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toUnmodifiableList());
-        return endpointCertificateProvider.requestCaSignedCertificate(instance.id(), dnsNamesOf(instance.id(), zones), currentMetadata);
+    private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) {
+
+        List<String> currentlyPresentNames = currentMetadata.isPresent() ?
+                currentMetadata.get().requestedDnsSans().orElseThrow(() -> new RuntimeException("Certificate metadata exists but SANs are not present!"))
+                : Collections.emptyList();
+
+        var requiredZones = new LinkedHashSet<>(Set.of(deploymentZone));
+
+        var zoneCandidateList = zoneRegistry.zones().controllerUpgraded().zones().stream().map(ZoneApi::getId).collect(Collectors.toList());
+
+        // If not deploying to a dev or perf zone, require all prod zones in deployment spec + test and staging
+        if (!deploymentZone.environment().isManuallyDeployed()) {
+            zoneCandidateList.stream()
+                    .filter(z -> z.environment().isTest() || instanceSpec.isPresent() && instanceSpec.get().deploysTo(Environment.prod, z.region()))
+                    .forEach(requiredZones::add);
+        }
+
+        var requiredNames = requiredZones.stream()
+                .flatMap(zone -> dnsNamesOf(instance.id(), zone).stream())
+                .collect(Collectors.toCollection(LinkedHashSet::new));
+
+        // Make sure all currently present names will remain present.
+        // Instead of just adding "currently present names", we regenerate them in case the names for a zone have changed.
+        zoneCandidateList.stream()
+                .map(zone -> dnsNamesOf(instance.id(), zone))
+                .filter(zoneNames -> zoneNames.stream().anyMatch(currentlyPresentNames::contains))
+                .filter(currentlyPresentNames::containsAll)
+                .forEach(requiredNames::addAll);
+
+        // This check must be relaxed if we ever remove from the set of names generated.
+        if (!requiredNames.containsAll(currentlyPresentNames))
+            throw new RuntimeException("SANs to be requested do not cover all existing names! Missing names: "
+                    + currentlyPresentNames.stream().filter(s -> !requiredNames.contains(s)).collect(Collectors.joining(", ")));
+
+        return endpointCertificateProvider.requestCaSignedCertificate(instance.id(), List.copyOf(requiredNames), currentMetadata);
     }
 
     private void validateEndpointCertificate(EndpointCertificateMetadata endpointCertificateMetadata, Instance instance, ZoneId zone) {
@@ -243,7 +280,7 @@ public class EndpointCertificateManager {
                         .filter(san -> san.getType().equals(SubjectAlternativeName.Type.DNS_NAME))
                         .map(SubjectAlternativeName::getValue).collect(Collectors.toSet());
 
-                var dnsNamesOfZone = dnsNamesOf(instance.id(), List.of(zone));
+                var dnsNamesOfZone = dnsNamesOf(instance.id(), zone);
                 if (!subjectAlternativeNames.containsAll(dnsNamesOfZone))
                     throw new EndpointCertificateException(EndpointCertificateException.Type.VERIFICATION_FAILURE, "Certificate is missing required SANs for zone " + zone.value());
 
@@ -259,22 +296,24 @@ public class EndpointCertificateManager {
             }
     }
 
-    private List<String> dnsNamesOf(ApplicationId applicationId, List<ZoneId> zones) {
+    private List<String> dnsNamesOf(ApplicationId applicationId, ZoneId zone) {
         List<String> endpointDnsNames = new ArrayList<>();
 
         // We add first an endpoint name based on a hash of the applicationId,
         // as the certificate provider requires the first CN to be < 64 characters long.
         endpointDnsNames.add(commonNameHashOf(applicationId, zoneRegistry.system()));
 
-        var globalDefaultEndpoint = Endpoint.of(applicationId).named(EndpointId.defaultId());
-        var rotationEndpoints = Endpoint.of(applicationId).wildcard();
+        List<Endpoint.EndpointBuilder> endpoints = new ArrayList<>();
+
+        if(zone.environment().isProduction()) {
+            endpoints.add(Endpoint.of(applicationId).named(EndpointId.defaultId()));
+            endpoints.add(Endpoint.of(applicationId).wildcard());
+        }
 
-        var zoneLocalEndpoints = zones.stream().flatMap(zone -> Stream.of(
-                Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone),
-                Endpoint.of(applicationId).wildcard(zone)
-        ));
+        endpoints.add(Endpoint.of(applicationId).target(ClusterSpec.Id.from("default"), zone));
+        endpoints.add(Endpoint.of(applicationId).wildcard(zone));
 
-        Stream.concat(Stream.of(globalDefaultEndpoint, rotationEndpoints), zoneLocalEndpoints)
+        endpoints.stream()
                 .map(endpoint -> endpoint.routingMethod(RoutingMethod.exclusive))
                 .map(endpoint -> endpoint.on(Endpoint.Port.tls()))
                 .map(endpointBuilder -> endpointBuilder.in(zoneRegistry.system()))
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 0ff14e01874..dfca2d69a5e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -700,6 +700,8 @@ public class ControllerTest {
         // Create app1
         var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
         var prodZone = ZoneId.from("prod", "us-west-1");
+        var stagingZone = ZoneId.from("staging", "us-east-3");
+        var testZone = ZoneId.from("test", "us-east-1");
         tester.controllerTester().zoneRegistry().exclusiveRoutingIn(ZoneApiMock.from(prodZone));
         var applicationPackage = new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
                                                                 .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION)
@@ -713,7 +715,7 @@ public class ControllerTest {
         assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud",
                                              "app1.tenant1.global.vespa.oath.cloud",
                                              "*.app1.tenant1.global.vespa.oath.cloud"),
-                                   tester.controller().zoneRegistry().zones().controllerUpgraded().ids().stream()
+                                   Stream.of(prodZone, testZone, stagingZone)
                                          .flatMap(zone -> Stream.of("", "*.")
                                                                 .map(prefix -> prefix + "app1.tenant1." + zone.region().value() +
                                                                                (zone.environment() == Environment.prod ? "" :  "." + zone.environment().value()) +
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
index d7bc73adf37..d0e87056821 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
@@ -1,6 +1,9 @@
 package com.yahoo.vespa.hosted.controller.certificate;
 
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.xml.DeploymentSpecXmlReader;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.security.KeyAlgorithm;
@@ -25,8 +28,10 @@ import java.security.cert.X509Certificate;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
+import java.util.Set;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
@@ -45,20 +50,49 @@ public class EndpointCertificateManagerTest {
     private final EndpointCertificateManager endpointCertificateManager =
             new EndpointCertificateManager(zoneRegistryMock, mockCuratorDb, secretStore, endpointCertificateMock, clock, inMemoryFlagSource);
 
+    private static List<String> expectedSans = List.of(
+            "vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud",
+            "default.default.global.vespa.oath.cloud",
+            "*.default.default.global.vespa.oath.cloud",
+            "default.default.aws-us-east-1a.vespa.oath.cloud",
+            "*.default.default.aws-us-east-1a.vespa.oath.cloud",
+            "default.default.us-east-1.test.vespa.oath.cloud",
+            "*.default.default.us-east-1.test.vespa.oath.cloud",
+            "default.default.us-east-3.staging.vespa.oath.cloud",
+            "*.default.default.us-east-3.staging.vespa.oath.cloud"
+    );
+
+    private static List<String> expectedAdditionalSans = List.of(
+            "default.default.ap-northeast-1.vespa.oath.cloud",
+            "*.default.default.ap-northeast-1.vespa.oath.cloud"
+    );
+
+    private static List<String> expectedCombinedSans = new ArrayList<>() {{
+        addAll(expectedSans);
+        addAll(expectedAdditionalSans);
+    }};
+
+    private static List<String> expectedDevSans = List.of(
+            "vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud",
+            "default.default.us-east-1.dev.vespa.oath.cloud",
+            "*.default.default.us-east-1.dev.vespa.oath.cloud"
+    );
+
     private static final KeyPair testKeyPair = KeyUtils.generateKeypair(KeyAlgorithm.EC, 192);
-    private static final X509Certificate testCertificate = X509CertificateBuilder
-            .fromKeypair(
-                    testKeyPair,
-                    new X500Principal("CN=test"),
-                    Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES),
-                    SignatureAlgorithm.SHA256_WITH_ECDSA,
-                    X509CertificateBuilder.generateRandomSerialNumber())
-            .addSubjectAlternativeName("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud")
-            .addSubjectAlternativeName("default.default.global.vespa.oath.cloud")
-            .addSubjectAlternativeName("*.default.default.global.vespa.oath.cloud")
-            .addSubjectAlternativeName("default.default.us-east-1.test.vespa.oath.cloud")
-            .addSubjectAlternativeName("*.default.default.us-east-1.test.vespa.oath.cloud")
-            .build();
+    private static final X509Certificate testCertificate = makeTestCert(expectedSans);
+    private static final X509Certificate testCertificate2 = makeTestCert(expectedCombinedSans);
+
+    private static X509Certificate makeTestCert(List<String> sans) {
+        X509CertificateBuilder x509CertificateBuilder = X509CertificateBuilder
+                .fromKeypair(
+                        testKeyPair,
+                        new X500Principal("CN=test"),
+                        Instant.now(), Instant.now().plus(5, ChronoUnit.MINUTES),
+                        SignatureAlgorithm.SHA256_WITH_ECDSA,
+                        X509CertificateBuilder.generateRandomSerialNumber());
+        for (String san : sans) x509CertificateBuilder = x509CertificateBuilder.addSubjectAlternativeName(san);
+        return x509CertificateBuilder.build();
+    }
 
     private final Instance testInstance = new Instance(ApplicationId.defaultId());
     private final String testKeyName = "testKeyName";
@@ -68,25 +102,37 @@ public class EndpointCertificateManagerTest {
     @Before
     public void setUp() {
         zoneRegistryMock.exclusiveRoutingIn(zoneRegistryMock.zones().all().zones());
-        testZone = zoneRegistryMock.zones().directlyRouted().zones().stream().findFirst().orElseThrow().getId();
+        testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().findFirst().orElseThrow().getId();
         inMemoryFlagSource.withBooleanFlag(Flags.VALIDATE_ENDPOINT_CERTIFICATES.id(), true);
     }
 
     @Test
-    public void provisions_new_certificate() {
-        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+    public void provisions_new_certificate_in_dev() {
+        ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.dev).zones().stream().findFirst().orElseThrow().getId();
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
+        assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
+        assertEquals(0, endpointCertificateMetadata.get().version());
+        assertEquals(expectedDevSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow());
+    }
+
+    @Test
+    public void provisions_new_certificate_in_prod() {
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
         assertTrue(endpointCertificateMetadata.isPresent());
         assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
         assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
         assertEquals(0, endpointCertificateMetadata.get().version());
+        assertEquals(expectedSans, endpointCertificateMetadata.get().requestedDnsSans().orElseThrow());
     }
 
     @Test
     public void reuses_stored_certificate_metadata() {
         mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7));
         secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 7);
-        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 7);
-        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 7);
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
         assertTrue(endpointCertificateMetadata.isPresent());
         assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
         assertEquals(testCertName, endpointCertificateMetadata.get().certName());
@@ -101,9 +147,9 @@ public class EndpointCertificateManagerTest {
         secretStore.setSecret(testCertName, "cert", 7);
         secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8);
         secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9);
-        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 8);
+        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8);
         mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7));
-        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
         assertTrue(endpointCertificateMetadata.isPresent());
         assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
         assertEquals(testCertName, endpointCertificateMetadata.get().certName());
@@ -114,11 +160,50 @@ public class EndpointCertificateManagerTest {
     public void reprovisions_certificate_when_necessary() {
         mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(List.of()), Optional.empty()));
         secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0);
-        secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate)+X509CertificateUtils.toPem(testCertificate), 0);
-        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone);
+        secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 0);
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
         assertTrue(endpointCertificateMetadata.isPresent());
         assertEquals(0, endpointCertificateMetadata.get().version());
         assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id()));
     }
 
+    @Test
+    public void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() {
+        ZoneId testZone = zoneRegistryMock.zones().directlyRouted().in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId();
+
+        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, Optional.of("uuid"), Optional.of(expectedSans), Optional.of("mockCa")));
+        secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1);
+        secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), -1);
+
+        secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0);
+        secretStore.setSecret("vespa.tls.default.default.default-cert", X509CertificateUtils.toPem(testCertificate2) + X509CertificateUtils.toPem(testCertificate2), 0);
+
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertEquals(0, endpointCertificateMetadata.get().version());
+        assertEquals(endpointCertificateMetadata, mockCuratorDb.readEndpointCertificateMetadata(testInstance.id()));
+        assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()));
+    }
+
+    @Test
+    public void includes_zones_in_deployment_spec_when_deploying_to_staging() {
+
+        DeploymentSpec deploymentSpec = new DeploymentSpecXmlReader(true).read(
+                "<deployment version=\"1.0\">\n" +
+                        "  <instance id=\"default\">\n" +
+                        "    <prod>\n" +
+                        "      <region active=\"true\">aws-us-east-1a</region>\n" +
+                        "      <region active=\"true\">ap-northeast-1</region>\n" +
+                        "    </prod>\n" +
+                        "  </instance>\n" +
+                        "</deployment>\n");
+
+        ZoneId testZone = zoneRegistryMock.zones().controllerUpgraded().in(Environment.staging).zones().stream().findFirst().orElseThrow().getId();
+        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.of(deploymentSpec.requireInstance("default")));
+        assertTrue(endpointCertificateMetadata.isPresent());
+        assertTrue(endpointCertificateMetadata.get().keyName().matches("vespa.tls.default.default.*-key"));
+        assertTrue(endpointCertificateMetadata.get().certName().matches("vespa.tls.default.default.*-cert"));
+        assertEquals(0, endpointCertificateMetadata.get().version());
+        assertEquals(Set.copyOf(expectedCombinedSans), Set.copyOf(endpointCertificateMetadata.get().requestedDnsSans().orElseThrow()));
+    }
 }