Merge branch 'master' into olaa/create-cloud-role

14 files changed, 103 insertions, 187 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
index 76fa52c0706..87531240752 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManager.java
@@ -1,6 +1,5 @@
 package com.yahoo.vespa.hosted.controller.certificate;
 
-import com.google.common.collect.Sets;
 import com.google.common.hash.Hashing;
 import com.google.common.io.BaseEncoding;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
@@ -14,19 +13,15 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.security.SubjectAlternativeName;
 import com.yahoo.security.X509CertificateUtils;
-import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.flags.BooleanFlag;
-import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
 import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateProvider;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneRegistry;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 import org.jetbrains.annotations.NotNull;
 
@@ -35,17 +30,12 @@ import java.security.cert.X509Certificate;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
-import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Collections;
-import java.util.HashSet;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Optional;
-import java.util.OptionalInt;
 import java.util.Set;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -54,6 +44,8 @@ import java.util.stream.Collectors;
  * Looks up stored endpoint certificate metadata, provisions new certificates if none is found,
  * re-provisions if zone is not covered, and uses refreshed certificates if a newer version is available.
  *
+ * See also EndpointCertificateMaintainer, which handles refreshes, deletions and triggers deployments
+ *
  * @author andreer
  */
 public class EndpointCertificateManager {
@@ -66,9 +58,6 @@ public class EndpointCertificateManager {
     private final EndpointCertificateProvider endpointCertificateProvider;
     private final Clock clock;
     private final BooleanFlag validateEndpointCertificates;
-    private final StringFlag deleteUnusedEndpointCertificates;
-    private final BooleanFlag endpointCertInSharedRouting;
-    private final BooleanFlag useEndpointCertificateMaintainer;
 
     public EndpointCertificateManager(ZoneRegistry zoneRegistry,
                                       CuratorDb curator,
@@ -81,16 +70,6 @@ public class EndpointCertificateManager {
         this.endpointCertificateProvider = endpointCertificateProvider;
         this.clock = clock;
         this.validateEndpointCertificates = Flags.VALIDATE_ENDPOINT_CERTIFICATES.bindTo(flagSource);
-        this.deleteUnusedEndpointCertificates = Flags.DELETE_UNUSED_ENDPOINT_CERTIFICATES.bindTo(flagSource);
-        this.endpointCertInSharedRouting = Flags.ENDPOINT_CERT_IN_SHARED_ROUTING.bindTo(flagSource);
-        this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(flagSource);
-        Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(() -> {
-            try {
-                this.deleteUnusedCertificates();
-            } catch (Throwable t) {
-                log.log(Level.INFO, "Unexpected Throwable caught while deleting unused endpoint certificates", t);
-            }
-        }, 1, 10, TimeUnit.MINUTES);
     }
 
     public Optional<EndpointCertificateMetadata> getEndpointCertificateMetadata(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) {
@@ -105,10 +84,6 @@ public class EndpointCertificateManager {
 
     @NotNull
     private Optional<EndpointCertificateMetadata> getOrProvision(Instance instance, ZoneId zone, Optional<DeploymentInstanceSpec> instanceSpec) {
-        boolean endpointCertInSharedRouting = this.endpointCertInSharedRouting.with(FetchVector.Dimension.APPLICATION_ID, instance.id().serializedForm()).value();
-        if (!zoneRegistry.zones().directlyRouted().ids().contains(zone) && !endpointCertInSharedRouting)
-            return Optional.empty();
-
         final var currentCertificateMetadata = curator.readEndpointCertificateMetadata(instance.id());
 
         if (currentCertificateMetadata.isEmpty()) {
@@ -130,74 +105,10 @@ public class EndpointCertificateManager {
             return Optional.of(reprovisionedCertificateMetadata);
         }
 
-        if (!useEndpointCertificateMaintainer.value()) {
-            // Look for and use refreshed certificate
-            var latestAvailableVersion = latestVersionInSecretStore(currentCertificateMetadata.get());
-            if (latestAvailableVersion.isPresent() && latestAvailableVersion.getAsInt() > currentCertificateMetadata.get().version()) {
-                var refreshedCertificateMetadata = currentCertificateMetadata.get()
-                        .withVersion(latestAvailableVersion.getAsInt())
-                        .withLastRefreshed(clock.instant().getEpochSecond());
-                validateEndpointCertificate(refreshedCertificateMetadata, instance, zone);
-                curator.writeEndpointCertificateMetadata(instance.id(), refreshedCertificateMetadata);
-                return Optional.of(refreshedCertificateMetadata);
-            }
-        }
-
         validateEndpointCertificate(currentCertificateMetadata.get(), instance, zone);
         return currentCertificateMetadata;
     }
 
-    enum CleanupMode {
-        DISABLE,
-        DRYRUN,
-        ENABLE
-    }
-
-    private void deleteUnusedCertificates() {
-        CleanupMode mode = CleanupMode.valueOf(deleteUnusedEndpointCertificates.value().toUpperCase());
-        if (mode == CleanupMode.DISABLE || useEndpointCertificateMaintainer.value()) return;
-
-        var oneMonthAgo = clock.instant().minus(30, ChronoUnit.DAYS);
-        curator.readAllEndpointCertificateMetadata().forEach((applicationId, storedMetaData) -> {
-            var lastRequested = Instant.ofEpochSecond(storedMetaData.lastRequested());
-            if (lastRequested.isBefore(oneMonthAgo) && hasNoDeployments(applicationId)) {
-                try (Lock lock = lock(applicationId)) {
-                    if (Optional.of(storedMetaData).equals(curator.readEndpointCertificateMetadata(applicationId))) {
-                        log.log(Level.INFO, "Cert for app " + applicationId.serializedForm()
-                                + " has not been requested in a month and app has no deployments"
-                                + (mode == CleanupMode.ENABLE ? ", deleting from provider and ZK" : ""));
-                        if (mode == CleanupMode.ENABLE) {
-                            endpointCertificateProvider.deleteCertificate(applicationId, storedMetaData);
-                            curator.deleteEndpointCertificateMetadata(applicationId);
-                        }
-                    }
-                }
-            }
-        });
-    }
-
-    private Lock lock(ApplicationId applicationId) {
-        return curator.lock(TenantAndApplicationId.from(applicationId));
-    }
-
-    private boolean hasNoDeployments(ApplicationId applicationId) {
-        var deployments = curator.readApplication(TenantAndApplicationId.from(applicationId))
-                .flatMap(app -> app.get(applicationId.instance()))
-                .map(Instance::deployments);
-
-        return deployments.isEmpty() || deployments.get().size() == 0;
-    }
-
-    private OptionalInt latestVersionInSecretStore(EndpointCertificateMetadata originalCertificateMetadata) {
-        try {
-            var certVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.certName()));
-            var keyVersions = new HashSet<>(secretStore.listSecretVersions(originalCertificateMetadata.keyName()));
-            return Sets.intersection(certVersions, keyVersions).stream().mapToInt(Integer::intValue).max();
-        } catch (SecretNotFoundException s) {
-            return OptionalInt.empty(); // Likely because the certificate is very recently provisioned - keep current version
-        }
-    }
-
     private EndpointCertificateMetadata provisionEndpointCertificate(Instance instance, Optional<EndpointCertificateMetadata> currentMetadata, ZoneId deploymentZone, Optional<DeploymentInstanceSpec> instanceSpec) {
 
         List<String> currentlyPresentNames = currentMetadata.isPresent() ?
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java
index 786547d4a67..8a48cbd281d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java
@@ -2,6 +2,7 @@
 package com.yahoo.vespa.hosted.controller.dns;
 
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
+import com.yahoo.yolean.Exceptions;
 
 import java.util.ArrayList;
 import java.util.Collection;
@@ -82,7 +83,7 @@ public class NameServiceQueue {
                 request.dispatchTo(nameService);
                 queue.requests.poll();
             } catch (Exception e) {
-                log.log(Level.WARNING, "Failed to execute " + request + ": " + e.getMessage() +
+                log.log(Level.WARNING, "Failed to execute " + request + ": " + Exceptions.toMessageString(e) +
                                           ", request will be retried");
             }
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
index 56b7e5b2e46..979cd9060d9 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/ControllerMaintenance.java
@@ -118,13 +118,13 @@ public class ControllerMaintenance extends AbstractComponent {
             this.outstandingChangeDeployer = duration(3, MINUTES);
             this.versionStatusUpdater = duration(3, MINUTES);
             this.readyJobsTrigger = duration(1, MINUTES);
-            this.deploymentMetricsMaintainer = duration(5, MINUTES);
+            this.deploymentMetricsMaintainer = duration(10, MINUTES);
             this.applicationOwnershipConfirmer = duration(12, HOURS);
-            this.systemUpgrader = duration(1, MINUTES);
+            this.systemUpgrader = duration(90, SECONDS);
             this.jobRunner = duration(90, SECONDS);
             this.osUpgrader = duration(1, MINUTES);
             this.contactInformationMaintainer = duration(12, HOURS);
-            this.nameServiceDispatcher = duration(10, SECONDS);
+            this.nameServiceDispatcher = duration(30, SECONDS);
             this.costReportMaintainer = duration(2, HOURS);
             this.resourceMeterMaintainer = duration(1, MINUTES);
             this.cloudEventReporter = duration(30, MINUTES);
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
index a1d7c3d16b4..e59875e9588 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainer.java
@@ -8,8 +8,6 @@ import com.yahoo.container.jdisc.secretstore.SecretNotFoundException;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.curator.Lock;
-import com.yahoo.vespa.flags.BooleanFlag;
-import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
@@ -27,12 +25,13 @@ import java.time.temporal.ChronoUnit;
 import java.util.HashSet;
 import java.util.Optional;
 import java.util.OptionalInt;
-import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
 /**
- * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates
+ * Updates refreshed endpoint certificates and triggers redeployment, and deletes unused certificates.
+ *
+ * See also EndpointCertificateManager, which provisions, reprovisions and validates certificates on deploy
  *
  * @author andreer
  */
@@ -45,7 +44,6 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
     private final CuratorDb curator;
     private final SecretStore secretStore;
     private final EndpointCertificateProvider endpointCertificateProvider;
-    private final BooleanFlag useEndpointCertificateMaintainer;
 
     public EndpointCertificateMaintainer(Controller controller, Duration interval) {
         super(controller, interval, null, SystemName.all());
@@ -54,15 +52,10 @@ public class EndpointCertificateMaintainer extends ControllerMaintainer {
         this.secretStore = controller.secretStore();
         this.curator = controller().curator();
         this.endpointCertificateProvider = controller.serviceRegistry().endpointCertificateProvider();
-        this.useEndpointCertificateMaintainer = Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.bindTo(controller().flagSource());
     }
 
     @Override
     protected boolean maintain() {
-
-        if (!useEndpointCertificateMaintainer.value())
-            return true; // handled by EndpointCertificateManager for now
-
         try {
             // In order of importance
             deployRefreshedCertificates();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
index d6739581c79..73528977166 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/JobRunner.java
@@ -80,10 +80,12 @@ public class JobRunner extends ControllerMaintainer {
     /** Advances each of the ready steps for the given run, or marks it as finished, and stashes it. Public for testing. */
     public void advance(Run run) {
         if (   ! run.hasFailed()
-            &&   controller().clock().instant().isAfter(run.start().plus(jobTimeout))) {
-            jobs.abort(run.id());
-            advance(jobs.run(run.id()).get());
-        }
+            &&   controller().clock().instant().isAfter(run.start().plus(jobTimeout)))
+            executors.execute(() -> {
+                jobs.abort(run.id());
+                advance(jobs.run(run.id()).get());
+            });
+
         else if (run.readySteps().isEmpty())
             executors.execute(() -> finish(run.id()));
         else
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 5a1496bf507..9331e5086cc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -2,6 +2,8 @@
 package com.yahoo.vespa.hosted.controller.restapi.application;
 
 import ai.vespa.hosted.api.Signatures;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.base.Joiner;
 import com.google.common.collect.ImmutableSet;
 import com.google.inject.Inject;
@@ -100,9 +102,6 @@ import com.yahoo.vespa.hosted.controller.versions.VersionStatus;
 import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 import com.yahoo.vespa.serviceview.bindings.ApplicationView;
 import com.yahoo.yolean.Exceptions;
-import org.json.JSONArray;
-import org.json.JSONException;
-import org.json.JSONObject;
 
 import javax.ws.rs.ForbiddenException;
 import javax.ws.rs.InternalServerErrorException;
@@ -151,6 +150,8 @@ import static java.util.stream.Collectors.toUnmodifiableList;
 @SuppressWarnings("unused") // created by injection
 public class ApplicationApiHandler extends LoggingRequestHandler {
 
+    private static final ObjectMapper jsonMapper = new ObjectMapper();
+
     private static final String OPTIONAL_PREFIX = "/api";
 
     private final Controller controller;
@@ -789,15 +790,15 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
     private JsonResponse buildResponseFromProtonMetrics(List<ProtonMetrics> protonMetrics) {
         try {
-            var jsonObject = new JSONObject();
-            var jsonArray = new JSONArray();
+            var jsonObject = jsonMapper.createObjectNode();
+            var jsonArray = jsonMapper.createArrayNode();
             for (ProtonMetrics metrics : protonMetrics) {
-                jsonArray.put(metrics.toJson());
+                jsonArray.add(metrics.toJson());
             }
-            jsonObject.put("metrics", jsonArray);
-            return new JsonResponse(200, jsonObject.toString());
-        } catch (JSONException e) {
-            log.severe("Unable to build JsonResponse with Proton data");
+            jsonObject.set("metrics", jsonArray);
+            return new JsonResponse(200, jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(jsonObject));
+        } catch (JsonProcessingException e) {
+            log.log(Level.SEVERE, "Unable to build JsonResponse with Proton data: " + e.getMessage(), e);
             return new JsonResponse(500, "");
         }
     }
@@ -1682,11 +1683,20 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
 
         controller.jobController().deploy(id, type, version, applicationPackage);
         RunId runId = controller.jobController().last(id, type).get().id();
+        DeploymentId deploymentId = new DeploymentId(id, type.zone(controller.system()));
+
         Slime slime = new Slime();
         Cursor rootObject = slime.setObject();
         rootObject.setString("message", "Deployment started in " + runId +
                                         ". This may take about 15 minutes the first time.");
         rootObject.setLong("run", runId.number());
+        var endpointArray = rootObject.setArray("endpoints");
+        EndpointList zoneEndpoints = controller.routing().endpointsOf(deploymentId)
+                .scope(Endpoint.Scope.zone)
+                .not().legacy();
+        for (var endpoint : controller.routing().directEndpoints(zoneEndpoints, deploymentId.applicationId())) {
+            toSlime(endpoint, endpointArray.addObject());
+        }
         return new SlimeJsonResponse(slime);
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 8fcbb365804..63452f40dbb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -736,8 +736,8 @@ public class ControllerTest {
                                                                 .map(prefix -> prefix + "app1.tenant1." + zone.region().value() +
                                                                                (zone.environment() == Environment.prod ? "" :  "." + zone.environment().value()) +
                                                                                ".vespa.oath.cloud")))
-                           .collect(Collectors.toUnmodifiableList()),
-                     tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId()));
+                           .collect(Collectors.toUnmodifiableSet()),
+                     Set.copyOf(tester.controllerTester().serviceRegistry().endpointCertificateMock().dnsNamesOf(context1.instanceId())));
 
         // Next deployment reuses certificate
         context1.submit(applicationPackage).deploy();
@@ -751,7 +751,7 @@ public class ControllerTest {
         tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none());
         assertTrue("Application deployed and activated",
                    tester.configServer().application(context2.instanceId(), devZone).get().activated());
-        assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent());
+        assertTrue("Provisions certificate also in zone with routing layer", certificate.apply(context2.instance()).isPresent());
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
index 4f2000b1902..0d70fe48a77 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificateManagerTest.java
@@ -146,27 +146,6 @@ public class EndpointCertificateManagerTest {
     }
 
     @Test
-    public void uses_refreshed_certificate_when_available_and_valid() {
-        secretStore.setSecret(testKeyName, "secret-key", 7);
-        secretStore.setSecret(testCertName, "cert", 7);
-        secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 8);
-        secretStore.setSecret(testKeyName, KeyUtils.toPem(testKeyPair.getPrivate()), 9);
-        secretStore.setSecret(testCertName, X509CertificateUtils.toPem(testCertificate) + X509CertificateUtils.toPem(testCertificate), 8);
-        mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, 7, 0, "request_id",
-                List.of("vt2ktgkqme5zlnp4tj4ttyor7fj3v7q5o.vespa.oath.cloud",
-                        "default.default.global.vespa.oath.cloud",
-                        "*.default.default.global.vespa.oath.cloud",
-                        "default.default.aws-us-east-1a.vespa.oath.cloud",
-                        "*.default.default.aws-us-east-1a.vespa.oath.cloud"),
-                "issuer", Optional.empty(), Optional.empty()));
-        Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(testInstance, testZone, Optional.empty());
-        assertTrue(endpointCertificateMetadata.isPresent());
-        assertEquals(testKeyName, endpointCertificateMetadata.get().keyName());
-        assertEquals(testCertName, endpointCertificateMetadata.get().certName());
-        assertEquals(8, endpointCertificateMetadata.get().version());
-    }
-
-    @Test
     public void reprovisions_certificate_when_necessary() {
         mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty()));
         secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), 0);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
index dbf102f23d7..66bda66bbf9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/EndpointCertificateMaintainerTest.java
@@ -2,15 +2,12 @@
 package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
-import com.yahoo.vespa.flags.Flags;
-import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.api.integration.certificates.EndpointCertificateMetadata;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.integration.SecretStoreMock;
-import org.junit.Before;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -33,11 +30,6 @@ public class EndpointCertificateMaintainerTest {
     private final EndpointCertificateMaintainer maintainer = new EndpointCertificateMaintainer(tester.controller(), Duration.ofHours(1));
     private final EndpointCertificateMetadata exampleMetadata = new EndpointCertificateMetadata("keyName", "certName", 0, 0, "uuid", List.of(), "issuer", Optional.empty(), Optional.empty());
 
-    @Before
-    public void setUp() throws Exception {
-        ((InMemoryFlagSource) tester.controller().flagSource()).withBooleanFlag(Flags.USE_ENDPOINT_CERTIFICATE_MAINTAINER.id(), true);
-    }
-
     @Test
     public void old_and_unused_cert_is_deleted() {
         tester.curator().writeEndpointCertificateMetadata(ApplicationId.defaultId(), exampleMetadata);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
index bda5a708a94..2bf6eb39089 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/ContainerTester.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.ApplicationName;
 import com.yahoo.container.http.filter.FilterChainRepository;
 import com.yahoo.jdisc.http.filter.SecurityRequestFilter;
 import com.yahoo.jdisc.http.filter.SecurityRequestFilterChain;
+import com.yahoo.test.json.JsonTestHelper;
 import com.yahoo.vespa.athenz.api.AthenzDomain;
 import com.yahoo.vespa.athenz.api.AthenzIdentity;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -65,6 +66,10 @@ public class ContainerTester {
                              .addRoleMember(action, identity);
     }
 
+    public void assertJsonResponse(Supplier<Request> request, File responseFile) {
+        assertResponse(request.get(), responseFile, 200, false, true);
+    }
+
     public void assertResponse(Supplier<Request> request, File responseFile) {
         assertResponse(request.get(), responseFile);
     }
@@ -82,6 +87,10 @@ public class ContainerTester {
     }
 
     public void assertResponse(Request request, File responseFile, int expectedStatusCode, boolean removeWhitespace) {
+        assertResponse(request, responseFile, expectedStatusCode, removeWhitespace, false);
+    }
+
+    private void assertResponse(Request request, File responseFile, int expectedStatusCode, boolean removeWhitespace, boolean compareJson) {
         String expectedResponse = readTestFile(responseFile.toString());
         expectedResponse = include(expectedResponse);
         if (removeWhitespace) expectedResponse = expectedResponse.replaceAll("(\"[^\"]*\")|\\s*", "$1"); // Remove whitespace
@@ -106,7 +115,11 @@ public class ContainerTester {
                         expectedResponsePattern, responseString);
             }
         } else {
-            assertEquals(responseFile.toString(), expectedResponse, responseString);
+            if (compareJson) {
+                JsonTestHelper.assertJsonEquals(expectedResponse, responseString);
+            } else {
+                assertEquals(responseFile.toString(), expectedResponse, responseString);
+            }
         }
         assertEquals("Status code", expectedStatusCode, response.getStatus());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 434c83898ee..6626134b69a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -242,7 +242,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/production-us-east-3/", POST)
                                       .data(entity)
                                       .userIdentity(HOSTED_VESPA_OPERATOR),
-                              "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}");
+                              "{\"message\":\"Deployment started in run 1 of production-us-east-3 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1," +
+                                      "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://instance1--application1--tenant1.us-east-3.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}");
         app1.runJob(JobType.productionUsEast3);
         tester.controller().applications().deactivate(app1.instanceId(), ZoneId.from("prod", "us-east-3"));
 
@@ -250,7 +251,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/deploy/dev-us-east-1/", POST)
                                       .data(entity)
                                       .userIdentity(USER_ID),
-                              "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1}");
+                              "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.instance1. This may take about 15 minutes the first time.\",\"run\":1," +
+                                      "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://instance1--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}");
         app1.runJob(JobType.devUsEast1);
 
         // GET dev application package
@@ -514,7 +516,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
         updateMetrics();
 
         // GET metrics
-        tester.assertResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-east-1/instance/default/metrics", GET)
+        tester.assertJsonResponse(request("/application/v4/tenant/tenant2/application/application1/environment/dev/region/us-east-1/instance/default/metrics", GET)
                         .userIdentity(USER_ID),
                                 new File("proton-metrics.json"));
 
@@ -1426,7 +1428,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/new-user/deploy/dev-us-east-1", POST)
                                       .data(entity)
                                       .userIdentity(userId),
-                              "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.new-user. This may take about 15 minutes the first time.\",\"run\":1}");
+                              "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for tenant1.application1.new-user. This may take about 15 minutes the first time.\",\"run\":1," +
+                                      "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://new-user--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}");
     }
 
     @Test
@@ -1471,7 +1474,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST)
                         .data(entity)
                         .userIdentity(developer),
-                "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":1}",
+                "{\"message\":\"Deployment started in run 1 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":1," +
+                        "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://myapp--sandbox.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}",
                 200);
 
         // To add temporary support allowing tenant admins to launch services
@@ -1482,7 +1486,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
         tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST)
                         .data(entity)
                         .userIdentity(developer2),
-                "{\"message\":\"Deployment started in run 2 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":2}",
+                "{\"message\":\"Deployment started in run 2 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":2," +
+                        "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://myapp--sandbox.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}",
                 200);
 
 
@@ -1491,7 +1496,8 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .data(applicationPackageInstance1.zippedContent())
                                       .contentType("application/zip")
                                       .userIdentity(developer2),
-                              "{\"message\":\"Deployment started in run 3 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":3}");
+                              "{\"message\":\"Deployment started in run 3 of dev-us-east-1 for sandbox.myapp. This may take about 15 minutes the first time.\",\"run\":3," +
+                                      "\"endpoints\":[{\"cluster\":\"default\",\"tls\":true,\"url\":\"https://myapp--sandbox.us-east-1.dev.vespa.oath.cloud:4443/\",\"scope\":\"zone\",\"routingMethod\":\"shared\"}]}");
 
         // POST (deploy) an application package not as content type application/zip — not multipart — is disallowed
         tester.assertResponse(request("/application/v4/tenant/sandbox/application/myapp/instance/default/deploy/dev-us-east-1", POST)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
index 828e2856cae..c43abf276c5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/JobControllerApiHandlerHelperTest.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.restapi.application;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.container.jdisc.HttpResponse;
+import com.yahoo.test.json.JsonTestHelper;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
@@ -12,8 +13,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.deployment.TestReport;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
-import org.json.JSONException;
-import org.json.JSONObject;
 import org.junit.Test;
 
 import java.io.ByteArrayOutputStream;
@@ -180,12 +179,10 @@ public class JobControllerApiHandlerHelperTest {
                        "jobs-direct-deployment.json");
     }
 
-    private void compare(HttpResponse response, String expected) throws JSONException, IOException {
+    private void compare(HttpResponse response, String expected) throws IOException {
         ByteArrayOutputStream baos = new ByteArrayOutputStream();
         response.render(baos);
-        JSONObject actualJSON = new JSONObject(new String(baos.toByteArray()));
-        JSONObject expectedJSON = new JSONObject(expected);
-        assertEquals(expectedJSON.toString(), actualJSON.toString());
+        JsonTestHelper.assertJsonEquals(expected, baos.toString());
     }
 
     private void assertResponse(HttpResponse response, String fileName) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json
index 8ea3f318d1d..c53cee8fd97 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/deployment-job-accepted-2.json
@@ -1,4 +1,13 @@
 {
   "message": "Deployment started in run 1 of dev-us-east-1 for tenant1.application1.myuser. This may take about 15 minutes the first time.",
-  "run": 1
+  "run": 1,
+  "endpoints": [
+    {
+      "cluster": "default",
+      "tls": true,
+      "url": "https://myuser--application1--tenant1.us-east-1.dev.vespa.oath.cloud:4443/",
+      "scope": "zone",
+      "routingMethod": "shared"
+    }
+  ]
 }
\ No newline at end of file
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json
index a7e5b3918d8..3fba9b3c91c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/proton-metrics.json
@@ -1,23 +1,26 @@
 {
-  "metrics": [{
-    "clusterId": "content/doc/",
-    "metrics": {
-      "resourceMemoryUsageAverage": 0.103482,
-      "documentsReadyCount": 11430,
-      "documentDiskUsage": 44021,
-      "resourceDiskUsageAverage": 0.0168421,
-      "documentsTotalCount": 11430,
-      "documentsActiveCount": 11430
+  "metrics": [
+    {
+      "clusterId": "content/doc/",
+      "metrics": {
+        "resourceMemoryUsageAverage": 0.103482,
+        "documentsReadyCount": 11430.0,
+        "documentDiskUsage": 44021.0,
+        "resourceDiskUsageAverage": 0.0168421,
+        "documentsTotalCount": 11430.0,
+        "documentsActiveCount": 11430.0
+      }
+    },
+    {
+      "clusterId": "content/music/",
+      "metrics": {
+        "resourceMemoryUsageAverage": 0.00912,
+        "documentsReadyCount": 32000.0,
+        "documentDiskUsage": 90113.0,
+        "resourceDiskUsageAverage": 0.23912,
+        "documentsTotalCount": 32210.0,
+        "documentsActiveCount": 32210.0
+      }
     }
-  }, {
-    "clusterId": "content/music/",
-    "metrics": {
-      "resourceMemoryUsageAverage": 0.00912,
-      "documentsReadyCount": 32000,
-      "documentDiskUsage": 90113,
-      "resourceDiskUsageAverage": 0.23912,
-      "documentsTotalCount": 32210,
-      "documentsActiveCount": 32210
-    }
-  }]
-}
\ No newline at end of file
+  ]
+}