Merge with master

24 files changed, 399 insertions, 271 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index c3d5788ccb3..b9b808d573e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -76,6 +77,7 @@ import java.security.Principal;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
@@ -86,6 +88,7 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.function.Consumer;
 import java.util.logging.Level;
 import java.util.logging.Logger;
@@ -208,7 +211,7 @@ public class ApplicationController {
         return findGlobalEndpoint(deployment).map(endpoint -> {
             try {
                 EndpointStatus status = configServer.getGlobalRotationStatus(deployment, endpoint.upstreamName());
-                return Collections.singletonMap(endpoint, status);
+                return Map.of(endpoint, status);
             } catch (IOException e) {
                 throw new UncheckedIOException("Failed to get rotation status of " + deployment, e);
             }
@@ -282,7 +285,6 @@ public class ApplicationController {
             ApplicationVersion applicationVersion;
             ApplicationPackage applicationPackage;
             Set<String> rotationNames = new HashSet<>();
-            Set<String> cnames;
 
             try (Lock lock = lock(applicationId)) {
                 LockedApplication application = new LockedApplication(require(applicationId), lock);
@@ -324,9 +326,9 @@ public class ApplicationController {
                 application = withRotation(application, zone);
                 Application app = application.get();
                 // Include global DNS names
-                cnames = app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).collect(Collectors.toSet());
+                app.endpointsIn(controller.system()).asList().stream().map(Endpoint::dnsName).forEach(rotationNames::add);
                 // Include rotation ID to ensure that deployment can respond to health checks with rotation ID as Host header
-                app.rotations().stream().map(RotationId::asString).forEach(cnames::add);
+                app.rotations().stream().map(RotationId::asString).forEach(rotationNames::add);
 
                 // Update application with information from application package
                 if (   ! preferOldestVersion
@@ -338,7 +340,7 @@ public class ApplicationController {
 
             // Carry out deployment without holding the application lock.
             options = withVersion(platformVersion, options);
-            ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames, cnames);
+            ActivateResult result = deploy(applicationId, applicationPackage, zone, options, rotationNames);
 
             lockOrThrow(applicationId, application ->
                     store(application.withNewDeployment(zone, applicationVersion, platformVersion, clock.instant(),
@@ -405,7 +407,7 @@ public class ApplicationController {
                     artifactRepository.getSystemApplicationPackage(application.id(), zone, version)
             );
             DeployOptions options = withVersion(version, DeployOptions.none());
-            return deploy(application.id(), applicationPackage, zone, options, Set.of(), Set.of());
+            return deploy(application.id(), applicationPackage, zone, options, Set.of());
         } else {
            throw new RuntimeException("This system application does not have an application package: " + application.id().toShortString());
         }
@@ -413,16 +415,15 @@ public class ApplicationController {
 
     /** Deploys the given tester application to the given zone. */
     public ActivateResult deployTester(TesterId tester, ApplicationPackage applicationPackage, ZoneId zone, DeployOptions options) {
-        return deploy(tester.id(), applicationPackage, zone, options, Collections.emptySet(), Collections.emptySet());
+        return deploy(tester.id(), applicationPackage, zone, options, Set.of());
     }
 
     private ActivateResult deploy(ApplicationId application, ApplicationPackage applicationPackage,
                                   ZoneId zone, DeployOptions deployOptions,
-                                  Set<String> rotationNames, Set<String> cnames) {
+                                  Set<String> rotationNames) {
         DeploymentId deploymentId = new DeploymentId(application, zone);
         ConfigServer.PreparedApplication preparedApplication =
-                configServer.deploy(deploymentId, deployOptions, cnames, rotationNames,
-                                    applicationPackage.zippedContent());
+                configServer.deploy(deploymentId, deployOptions, rotationNames, List.of(), applicationPackage.zippedContent());
 
         // Refresh routing policies on successful deployment. At this point we can safely assume that the config server
         // has allocated load balancers for the deployment.
@@ -466,8 +467,8 @@ public class ApplicationController {
         logEntry.message = "Ignoring deployment of application '" + application + "' to " + zone +
                            " as a deployment is not currently expected";
         PrepareResponse prepareResponse = new PrepareResponse();
-        prepareResponse.log = Collections.singletonList(logEntry);
-        prepareResponse.configChangeActions = new ConfigChangeActions(Collections.emptyList(), Collections.emptyList());
+        prepareResponse.log = List.of(logEntry);
+        prepareResponse.configChangeActions = new ConfigChangeActions(List.of(), List.of());
         return new ActivateResult(new RevisionId("0"), prepareResponse, 0);
     }
 
@@ -518,26 +519,59 @@ public class ApplicationController {
         controller.nameServiceForwarder().createCname(RecordName.from(name), RecordData.fqdn(targetName), Priority.normal);
     }
 
-    /** Returns the endpoints of the deployment, or an empty list if the request fails */
-    public Optional<List<URI>> getDeploymentEndpoints(DeploymentId deploymentId) {
+    /** Returns the endpoints of the deployment, or empty if the request fails */
+    public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) {
         if ( ! get(deploymentId.applicationId())
                 .map(application -> application.deployments().containsKey(deploymentId.zoneId()))
                 .orElse(deploymentId.applicationId().instance().isTester()))
             throw new NotExistsException("Deployment", deploymentId.toString());
 
         try {
-            return Optional.of(ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
-                                                                    .map(RoutingEndpoint::endpoint)
-                                                                    .map(URI::create)
-                                                                    .iterator()));
+            return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
+                                                        .map(RoutingEndpoint::endpoint)
+                                                        .map(URI::create)
+                                                        .iterator());
         }
         catch (RuntimeException e) {
             log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId + ": "
                                    + Exceptions.toMessageString(e));
-            return Optional.empty();
+            return Collections.emptyList();
         }
     }
 
+    /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */
+    public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) {
+        if ( ! get(id.applicationId())
+                .map(application -> application.deployments().containsKey(id.zoneId()))
+                .orElse(id.applicationId().instance().isTester()))
+            throw new NotExistsException("Deployment", id.toString());
+
+        try {
+            return Optional.of(routingGenerator.clusterEndpoints(id))
+                    .filter(endpoints -> ! endpoints.isEmpty())
+                    .orElseGet(() -> routingPolicies.get(id).stream()
+                                                    .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone)
+                                                    .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(),
+                                                                                          policy -> policy.endpointIn(controller.system()).url())));
+        }
+        catch (RuntimeException e) {
+            log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": "
+                                   + Exceptions.toMessageString(e));
+            return Collections.emptyMap();
+        }
+    }
+
+    /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */
+    public Map<ZoneId, Map<ClusterSpec.Id, URI>> clusterEndpoints(ApplicationId id, Collection<ZoneId> zones) {
+        Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value));
+        for (ZoneId zone : zones) {
+            var endpoints = clusterEndpoints(new DeploymentId(id, zone));
+            if ( ! endpoints.isEmpty())
+                deployments.put(zone, endpoints);
+        }
+        return Collections.unmodifiableMap(deployments);
+    }
+
     /**
      * Deletes the the given application. All known instances of the applications will be deleted,
      * including PR instances.
@@ -778,7 +812,7 @@ public class ApplicationController {
             if (!"warn".equalsIgnoreCase(log.level) && !"warning".equalsIgnoreCase(log.level)) continue;
             warnings.merge(DeploymentMetrics.Warning.all, 1, Integer::sum);
         }
-        return Collections.unmodifiableMap(warnings);
+        return Map.copyOf(warnings);
     }
 
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 6c9b8dd0784..5026ca75a83 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -152,7 +152,6 @@ public class Endpoint {
                 if (legacy) return YAHOO_DNS_SUFFIX;
                 return OATH_DNS_SUFFIX;
             case Public:
-            case vaas:
                 return PUBLIC_DNS_SUFFIX;
             case PublicCd:
                 return PUBLIC_CD_DNS_SUFFIX;
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 891a696be9c..bf2460284ab 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -11,13 +11,11 @@ import com.yahoo.config.application.api.Notifications.When;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
@@ -28,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -41,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.io.PrintStream;
 import java.io.UncheckedIOException;
 import java.net.URI;
@@ -57,7 +53,6 @@ import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static com.yahoo.config.application.api.Notifications.Role.author;
 import static com.yahoo.config.application.api.Notifications.When.failing;
@@ -99,10 +94,12 @@ public class InternalStepRunner implements StepRunner {
     static final Duration installationTimeout = Duration.ofMinutes(150);
 
     private final Controller controller;
+    private final TestConfigSerializer testConfigSerializer;
     private final DeploymentFailureMails mails;
 
     public InternalStepRunner(Controller controller) {
         this.controller = controller;
+        this.testConfigSerializer = new TestConfigSerializer(controller.system());
         this.mails = new DeploymentFailureMails(controller.zoneRegistry());
     }
 
@@ -320,19 +317,23 @@ public class InternalStepRunner implements StepRunner {
 
     private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) {
         logger.log("Attempting to find deployment endpoints ...");
-        Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id, Set.of(zoneId));
+        var endpoints = controller.applications().clusterEndpoints(id, Set.of(zoneId));
         if ( ! endpoints.containsKey(zoneId)) {
             logger.log("Endpoints not yet ready.");
             return false;
         }
+        logEndpoints(endpoints, logger);
+        return true;
+    }
+
+    private void logEndpoints(Map<ZoneId, Map<ClusterSpec.Id, URI>> endpoints, DualLogger logger) {
         List<String> messages = new ArrayList<>();
         messages.add("Found endpoints:");
         endpoints.forEach((zone, uris) -> {
             messages.add("- " + zone);
-            uris.forEach(uri -> messages.add(" |-- " + uri));
+            uris.forEach((cluster, uri) -> messages.add(" |-- " + uri + " (" + cluster + ")"));
         });
         logger.log(messages);
-        return true;
     }
 
     private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) {
@@ -384,21 +385,15 @@ public class InternalStepRunner implements StepRunner {
             return Optional.of(aborted);
         }
 
-        Set<ZoneId> zones = testedZoneAndProductionZones(id);
+        Set<ZoneId> zones = controller.jobController().testedZoneAndProductionZones(id.application(), id.type());
 
         logger.log("Attempting to find endpoints ...");
-        Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id.application(), zones);
+        var endpoints = controller.applications().clusterEndpoints(id.application(), zones);
         if ( ! endpoints.containsKey(id.type().zone(controller.system())) && timedOut(deployment.get(), endpointTimeout)) {
             logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!");
             return Optional.of(error);
         }
-        List<String> messages = new ArrayList<>();
-        messages.add("Found endpoints:");
-        endpoints.forEach((zone, uris) -> {
-            messages.add("- " + zone);
-            uris.forEach(uri -> messages.add(" |-- " + uri));
-        });
-        logger.log(messages);
+        logEndpoints(endpoints, logger);
 
         Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
         if (testerEndpoint.isEmpty() && timedOut(deployment.get(), endpointTimeout)) {
@@ -410,9 +405,10 @@ public class InternalStepRunner implements StepRunner {
             logger.log("Starting tests ...");
             controller.jobController().cloud().startTests(testerEndpoint.get(),
                                                           TesterCloud.Suite.of(id.type()),
-                                                          testConfig(id.application(), id.type().zone(controller.system()),
-                                                                     controller.system(), endpoints,
-                                                                     listClusters(id.application(), zones)));
+                                                          testConfigSerializer.configJson(id.application(),
+                                                                                          id.type(),
+                                                                                          endpoints,
+                                                                                          listClusters(id.application(), zones)));
             return Optional.of(running);
         }
 
@@ -612,28 +608,6 @@ public class InternalStepRunner implements StepRunner {
         throw new IllegalStateException("No step deploys to the zone this run is for!");
     }
 
-    /** Returns a stream containing the zone of the deployment tested in the given run, and all production zones for the application. */
-    private Set<ZoneId> testedZoneAndProductionZones(RunId id) {
-        return Stream.concat(Stream.of(id.type().zone(controller.system())),
-                             application(id.application()).productionDeployments().keySet().stream())
-                     .collect(Collectors.toSet());
-    }
-
-    /** Returns all endpoints for all current deployments of the given real application. */
-    private Map<ZoneId, List<URI>> deploymentEndpoints(ApplicationId id, Iterable<ZoneId> zones) {
-        ImmutableMap.Builder<ZoneId, List<URI>> deployments = ImmutableMap.builder();
-        for (ZoneId zone : zones) {
-            controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone))
-                      .filter(endpoints -> ! endpoints.isEmpty())
-                      .or(() -> Optional.of(controller.applications().routingPolicies().get(id, zone).stream()
-                                                      .map(policy -> policy.endpointIn(controller.system()).url())
-                                                      .collect(Collectors.toUnmodifiableList()))
-                                        .filter(endpoints -> ! endpoints.isEmpty()))
-                      .ifPresent(endpoints -> deployments.put(zone, endpoints));
-        }
-        return deployments.build();
-    }
-
     /** Returns all content clusters in all current deployments of the given real application. */
     private Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) {
         ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder();
@@ -712,38 +686,6 @@ public class InternalStepRunner implements StepRunner {
         return deploymentSpec.getBytes(StandardCharsets.UTF_8);
     }
 
-    /** Returns the config for the tests to run for the given job. */
-    private static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system,
-                                     Map<ZoneId, List<URI>> deployments, Map<ZoneId, List<String>> clusters) {
-        Slime slime = new Slime();
-        Cursor root = slime.setObject();
-
-        root.setString("application", id.serializedForm());
-        root.setString("zone", testerZone.value());
-        root.setString("system", system.value());
-
-        Cursor endpointsObject = root.setObject("endpoints");
-        deployments.forEach((zone, endpoints) -> {
-            Cursor endpointArray = endpointsObject.setArray(zone.value());
-            for (URI endpoint : endpoints)
-                endpointArray.addString(endpoint.toString());
-        });
-
-        Cursor clustersObject = root.setObject("clusters");
-        clusters.forEach((zone, clusterList) -> {
-            Cursor clusterArray = clustersObject.setArray(zone.value());
-            for (String cluster : clusterList)
-                clusterArray.addString(cluster);
-        });
-
-        try {
-            return SlimeUtils.toJsonBytes(slime);
-        }
-        catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */
     private class DualLogger {
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 5bb9686ce0f..c644af2e554 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -41,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.google.common.collect.ImmutableList.copyOf;
@@ -408,12 +410,19 @@ public class JobController {
     Optional<URI> testerEndpoint(RunId id) {
         DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system()));
         return controller.applications().getDeploymentEndpoints(testerId)
-                         .flatMap(uris -> uris.stream().findAny())
+                         .stream().findAny()
                          .or(() -> controller.applications().routingPolicies().get(testerId).stream()
                                              .findAny()
                                              .map(policy -> policy.endpointIn(controller.system()).url()));
     }
 
+    /** Returns a set containing the zone of the deployment tested in the given run, and all production zones for the application. */
+    public Set<ZoneId> testedZoneAndProductionZones(ApplicationId id, JobType type) {
+        return Stream.concat(Stream.of(type.zone(controller.system())),
+                             controller.applications().require(id).productionDeployments().keySet().stream())
+                     .collect(Collectors.toSet());
+    }
+
     // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service.
     private long nextBuild(ApplicationId id) {
         return 1 + controller.applications().require(id).deploymentJobs()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
new file mode 100644
index 00000000000..e79692d34ed
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
@@ -0,0 +1,82 @@
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Serializes config for integration tests against Vespa deployments.
+ *
+ * @author jonmv
+ */
+public class TestConfigSerializer {
+
+    private final SystemName system;
+
+    public TestConfigSerializer(SystemName system) {
+        this.system = system;
+    }
+
+    public Slime configSlime(ApplicationId id,
+                             JobType type,
+                             Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments,
+                             Map<ZoneId, List<String>> clusters) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+
+        root.setString("application", id.serializedForm());
+        root.setString("zone", type.zone(system).value());
+        root.setString("system", system.value());
+
+        Cursor endpointsObject = root.setObject("endpoints"); // TODO jvenstad: remove.
+        deployments.forEach((zone, endpoints) -> {
+            Cursor endpointArray = endpointsObject.setArray(zone.value());
+            for (URI endpoint : endpoints.values())
+                endpointArray.addString(endpoint.toString());
+        });
+
+        Cursor zoneEndpointsObject = root.setObject("zoneEndpoints");
+        deployments.forEach((zone, endpoints) -> {
+            Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value());
+            endpoints.forEach((cluster, endpoint) -> {
+                clusterEndpointsObject.setString(cluster.value(), endpoint.toString());
+            });
+        });
+
+        if ( ! clusters.isEmpty()) {
+            Cursor clustersObject = root.setObject("clusters");
+            clusters.forEach((zone, clusterList) -> {
+                Cursor clusterArray = clustersObject.setArray(zone.value());
+                for (String cluster : clusterList)
+                    clusterArray.addString(cluster);
+            });
+        }
+
+        return slime;
+    }
+
+    /** Returns the config for the tests to run for the given job. */
+    public byte[] configJson(ApplicationId id,
+                             JobType type,
+                             Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments,
+                             Map<ZoneId, List<String>> clusters) {
+        try {
+            return SlimeUtils.toJsonBytes(configSlime(id, type, deployments, clusters));
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java
index 299ea168c7a..4e461534cc0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java
@@ -78,6 +78,7 @@ public class NameServiceForwarder {
                                           "requests. This likely means that the name service is not successfully " +
                                           "executing requests");
             }
+            log.log(LogLevel.INFO, "Queueing name service request: " + request);
             db.writeNameServiceQueue(queue.with(request, priority).last(maxQueuedRequests));
         }
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java
index 684fb091d92..4768577aa7b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceQueue.java
@@ -74,6 +74,7 @@ public class NameServiceQueue {
         var queue = new NameServiceQueue(requests);
         for (int i = 0; i < n && !queue.requests.isEmpty(); i++) {
             var request = queue.requests.peek();
+            log.log(LogLevel.INFO, "Dispatching name service request: " + request);
             try {
                 request.dispatchTo(nameService);
                 queue.requests.poll();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java
index 508401b0e14..4a98cb49227 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicies.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
@@ -74,46 +75,44 @@ public class RoutingPolicies {
         if (!controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) return;
         var lbs = new LoadBalancers(application, zone, controller.applications().configServer()
                                                                  .getLoadBalancers(application, zone));
-        removeObsoleteEndpointsFromDns(lbs);
-        storePoliciesOf(lbs);
-        removeObsoletePolicies(lbs);
-        registerEndpointsInDns(lbs);
+        try (var lock = db.lockRoutingPolicies()) {
+            removeObsoleteEndpointsFromDns(lbs, lock);
+            storePoliciesOf(lbs, lock);
+            removeObsoletePolicies(lbs, lock);
+            registerEndpointsInDns(lbs, lock);
+        }
     }
 
     /** Create global endpoints for given route, if any */
-    private void registerEndpointsInDns(LoadBalancers loadBalancers) {
-        try (var lock = db.lockRoutingPolicies()) {
-            Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application));
-
-            // Create DNS record for each routing ID
-            for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
-                Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().rotation(),
-                                                             controller.system());
-                Set<AliasTarget> targets = routeEntry.getValue()
-                                                     .stream()
-                                                     .filter(policy -> policy.dnsZone().isPresent())
-                                                     .map(policy -> new AliasTarget(policy.canonicalName(),
-                                                                                    policy.dnsZone().get(),
-                                                                                    policy.zone()))
-                                                     .collect(Collectors.toSet());
-                controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal);
-            }
+    private void registerEndpointsInDns(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+        Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(get(loadBalancers.application));
+
+        // Create DNS record for each routing ID
+        for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
+            Endpoint endpoint = RoutingPolicy.endpointOf(routeEntry.getKey().application(), routeEntry.getKey().rotation(),
+                                                         controller.system());
+            Set<AliasTarget> targets = routeEntry.getValue()
+                                                 .stream()
+                                                 .filter(policy -> policy.dnsZone().isPresent())
+                                                 .map(policy -> new AliasTarget(policy.canonicalName(),
+                                                                                policy.dnsZone().get(),
+                                                                                policy.zone()))
+                                                 .collect(Collectors.toSet());
+            controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()), targets, Priority.normal);
         }
     }
 
     /** Store routing policies for given route */
-    private void storePoliciesOf(LoadBalancers loadBalancers) {
-        try (var lock = db.lockRoutingPolicies()) {
-            Set<RoutingPolicy> policies = new LinkedHashSet<>(get(loadBalancers.application));
-            for (LoadBalancer loadBalancer : loadBalancers.list) {
-                RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer);
-                if (!policies.add(policy)) {
-                    policies.remove(policy);
-                    policies.add(policy);
-                }
+    private void storePoliciesOf(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+        Set<RoutingPolicy> policies = new LinkedHashSet<>(get(loadBalancers.application));
+        for (LoadBalancer loadBalancer : loadBalancers.list) {
+            RoutingPolicy policy = createPolicy(loadBalancers.application, loadBalancers.zone, loadBalancer);
+            if (!policies.add(policy)) {
+                policies.remove(policy);
+                policies.add(policy);
             }
-            db.writeRoutingPolicies(loadBalancers.application, policies);
         }
+        db.writeRoutingPolicies(loadBalancers.application, policies);
     }
 
     /** Create a policy for given load balancer and register a CNAME for it */
@@ -128,36 +127,32 @@ public class RoutingPolicies {
     }
 
     /** Remove obsolete policies for given route and their CNAME records */
-    private void removeObsoletePolicies(LoadBalancers loadBalancers) {
-        try (var lock = db.lockRoutingPolicies()) {
-            var allPolicies = new LinkedHashSet<>(get(loadBalancers.application));
-            var removalCandidates = new HashSet<>(allPolicies);
-            var activeLoadBalancers = loadBalancers.list.stream()
-                                                        .map(LoadBalancer::hostname)
-                                                        .collect(Collectors.toSet());
-            // Remove active load balancers and irrelevant zones from candidates
-            removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) ||
-                                                 !policy.zone().equals(loadBalancers.zone));
-            for (var policy : removalCandidates) {
-                var dnsName = policy.endpointIn(controller.system()).dnsName();
-                controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal);
-                allPolicies.remove(policy);
-            }
-            db.writeRoutingPolicies(loadBalancers.application, allPolicies);
+    private void removeObsoletePolicies(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+        var allPolicies = new LinkedHashSet<>(get(loadBalancers.application));
+        var removalCandidates = new HashSet<>(allPolicies);
+        var activeLoadBalancers = loadBalancers.list.stream()
+                                                    .map(LoadBalancer::hostname)
+                                                    .collect(Collectors.toSet());
+        // Remove active load balancers and irrelevant zones from candidates
+        removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()) ||
+                                             !policy.zone().equals(loadBalancers.zone));
+        for (var policy : removalCandidates) {
+            var dnsName = policy.endpointIn(controller.system()).dnsName();
+            controller.nameServiceForwarder().removeRecords(Record.Type.CNAME, RecordName.from(dnsName), Priority.normal);
+            allPolicies.remove(policy);
         }
+        db.writeRoutingPolicies(loadBalancers.application, allPolicies);
     }
 
     /** Remove unreferenced global endpoints for given route from DNS */
-    private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers) {
-        try (var lock = db.lockRoutingPolicies()) {
-            var zonePolicies = get(loadBalancers.application, loadBalancers.zone);
-            var removalCandidates = routingTableFrom(zonePolicies).keySet();
-            var activeRoutingIds = routingIdsFrom(loadBalancers.list);
-            removalCandidates.removeAll(activeRoutingIds);
-            for (var id : removalCandidates) {
-                Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller.system());
-                controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal);
-            }
+    private void removeObsoleteEndpointsFromDns(LoadBalancers loadBalancers, @SuppressWarnings("unused") Lock lock) {
+        var zonePolicies = get(loadBalancers.application, loadBalancers.zone);
+        var removalCandidates = routingTableFrom(zonePolicies).keySet();
+        var activeRoutingIds = routingIdsFrom(loadBalancers.list);
+        removalCandidates.removeAll(activeRoutingIds);
+        for (var id : removalCandidates) {
+            Endpoint endpoint = RoutingPolicy.endpointOf(id.application(), id.rotation(), controller.system());
+            controller.nameServiceForwarder().removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()), Priority.normal);
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
index a2e5a0c78f7..d704d701cf0 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/CuratorDb.java
@@ -106,6 +106,10 @@ public class CuratorDb {
     CuratorDb(Curator curator, Duration tryLockTimeout) {
         this.curator = curator;
         this.tryLockTimeout = tryLockTimeout;
+
+        // TODO: Remove after 7.60
+        curator.delete(root.append("openStackServerPool"));
+        curator.delete(root.append("vespaServerPool"));
     }
 
     /** Returns all hosts configured to be part of this ZooKeeper cluster */
@@ -168,16 +172,6 @@ public class CuratorDb {
         return lock(lockPath(provisionStateId), Duration.ofSeconds(1));
     }
 
-    @SuppressWarnings("unused") // Called by internal code
-    public Lock lockVespaServerPool() {
-        return lock(lockRoot.append("vespaServerPoolLock"), Duration.ofSeconds(1));
-    }
-
-    @SuppressWarnings("unused") // Called by internal code
-    public Lock lockOpenStackServerPool() {
-        return lock(lockRoot.append("openStackServerPoolLock"), Duration.ofSeconds(1));
-    }
-
     public Lock lockOsVersions() {
         return lock(lockRoot.append("osTargetVersion"), defaultLockTimeout);
     }
@@ -469,26 +463,6 @@ public class CuratorDb {
         return curator.getChildren(provisionStatePath());
     }
 
-    @SuppressWarnings("unused")
-    public Optional<byte[]> readVespaServerPool() {
-        return curator.getData(vespaServerPoolPath());
-    }
-
-    @SuppressWarnings("unused")
-    public void writeVespaServerPool(byte[] data) {
-        curator.set(vespaServerPoolPath(), data);
-    }
-
-    @SuppressWarnings("unused")
-    public Optional<byte[]> readOpenStackServerPool() {
-        return curator.getData(openStackServerPoolPath());
-    }
-
-    @SuppressWarnings("unused")
-    public void writeOpenStackServerPool(byte[] data) {
-        curator.set(openStackServerPoolPath(), data);
-    }
-
     // -------------- Routing policies ----------------------------------------
 
     public void writeRoutingPolicies(ApplicationId application, Set<RoutingPolicy> policies) {
@@ -589,14 +563,6 @@ public class CuratorDb {
         return provisionStatePath().append(provisionId);
     }
 
-    private static Path vespaServerPoolPath() {
-        return root.append("vespaServerPool");
-    }
-
-    private static Path openStackServerPoolPath() {
-        return root.append("openStackServerPool");
-    }
-
     private static Path tenantPath(TenantName name) {
         return tenantRoot.append(name.value());
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index b4483010176..3db8c447572 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -66,6 +66,7 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel;
+import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
@@ -95,6 +96,7 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -186,6 +188,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/deploying/pin")) return deploying(path.get("tenant"), path.get("application"), path.get("instance"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/environment/{environment}/region/{region}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -629,7 +632,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         // ask the routing layer here
         Cursor serviceUrlArray = response.setArray("serviceUrls");
         controller.applications().getDeploymentEndpoints(deploymentId)
-                  .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())));
+                  .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()));
 
         response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString());
         response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString());
@@ -1139,6 +1142,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
     }
 
+    private HttpResponse testConfig(ApplicationId id, JobType type) {
+        var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type));
+        return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id,
+                                                                                               type,
+                                                                                               endpoints,
+                                                                                               Collections.emptyMap()));
+    }
+
     private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) {
         Optional<DeploymentJobs.JobError> jobError = Optional.empty();
         if (report.field("jobError").valid()) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 3fca94ef21f..de31f1f67f9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -293,7 +293,7 @@ public class ControllerTest {
                                 "app1--tenant1.global.vespa.oath.cloud",
                                 "app1.tenant1.global.vespa.yahooapis.com",
                                 "app1--tenant1.global.vespa.yahooapis.com"),
-                         tester.configServer().rotationCnames().get(new DeploymentId(application.id(), deployment.zone())));
+                         tester.configServer().rotationNames().get(new DeploymentId(application.id(), deployment.zone())));
         }
         tester.flushDnsRequests();
         assertEquals(3, tester.controllerTester().nameService().records().size());
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 370fd03d9e7..095651df033 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
@@ -25,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.URI;
@@ -34,6 +38,7 @@ import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Executors;
@@ -369,7 +374,7 @@ public class InternalStepRunnerTest {
         tester.runner().run();
         assertEquals(failed, tester.jobs().run(id).get().steps().get(Step.endTests));
         assertTestLogEntries(id, Step.copyVespaLogs,
-                             new LogEntry(lastId + 2, tester.clock().millis(), debug, "Copying Vespa log from nodes of tenant.application in zone test.us-east-1 in default ..."),
+                             new LogEntry(lastId + 2, tester.clock().millis(), debug, "Copying Vespa log from nodes of tenant.application in test.us-east-1 ..."),
                              new LogEntry(lastId + 3, 1554970337084L, info,
                                           "17480180-v6-3.ostk.bm2.prod.ne1.yahoo.com\tcontainer\tContainer.com.yahoo.container.jdisc.ConfiguredApplication\n" +
                                           "Switching to the latest deployed set of configurations and components. Application switch number: 2"),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
new file mode 100644
index 00000000000..bc411d4377d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
@@ -0,0 +1,36 @@
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class TestConfigSerializerTest {
+
+    @Test
+    public void testConfig() throws IOException {
+        ZoneId zone = JobType.systemTest.zone(SystemName.PublicCd);
+        byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(appId,
+                                                                               JobType.systemTest,
+                                                                               Map.of(zone, Map.of(ClusterSpec.Id.from("ai"),
+                                                                                                   URI.create("https://server/"))),
+                                                                               Map.of(zone, List.of("facts")));
+        byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes();
+        assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))),
+                     new String(json));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 1894a51adc3..d4df9c20ead 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -16,6 +16,7 @@ import com.yahoo.vespa.hosted.controller.api.identifiers.Hostname;
 import com.yahoo.vespa.hosted.controller.api.identifiers.Identifier;
 import com.yahoo.vespa.hosted.controller.api.identifiers.TenantId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServer;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.ContainerEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Log;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
@@ -59,7 +60,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private final Set<DeploymentId> suspendedApplications = new HashSet<>();
     private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>();
     private final Map<DeploymentId, List<Log>> warnings = new HashMap<>();
-    private final Map<DeploymentId, Set<String>> rotationCnames = new HashMap<>();
+    private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>();
 
     private Version lastPrepareVersion = null;
     private RuntimeException prepareException = null;
@@ -180,8 +181,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         warnings.put(deployment, List.copyOf(logs));
     }
 
-    public Map<DeploymentId, Set<String>> rotationCnames() {
-        return Collections.unmodifiableMap(rotationCnames);
+    public Map<DeploymentId, Set<String>> rotationNames() {
+        return Collections.unmodifiableMap(rotationNames);
     }
 
     @Override
@@ -223,8 +224,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     }
 
     @Override
-    public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationCnames,
-                                       Set<String> rotationNames, byte[] content) {
+    public PreparedApplication deploy(DeploymentId deployment, DeployOptions deployOptions, Set<String> rotationNames,
+                                      List<ContainerEndpoint> containerEndpoints, byte[] content) {
         lastPrepareVersion = deployOptions.vespaVersion.map(Version::fromString).orElse(null);
         if (prepareException != null) {
             RuntimeException prepareException = this.prepareException;
@@ -236,64 +237,41 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         if (nodeRepository().list(deployment.zoneId(), deployment.applicationId()).isEmpty())
             provision(deployment.zoneId(), deployment.applicationId());
 
-        this.rotationCnames.put(deployment, Set.copyOf(rotationCnames));
-
-        return new PreparedApplication() {
-
-            // TODO: Remove when no longer part of interface
-            public void activate() {}
-
-            // TODO: Remove when no longer part of interface
-            public List<Log> messages() {
-                Log warning = new Log();
-                warning.level = "WARNING";
-                warning.time  = 1;
-                warning.message = "The warning";
-
-                Log info = new Log();
-                info.level = "INFO";
-                info.time  = 2;
-                info.message = "The info";
-
-                return List.of(warning, info);
-            }
-
-            @Override
-            public PrepareResponse prepareResponse() {
-                Application application = applications.get(deployment.applicationId());
-                application.activate();
-                List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId());
-                for (Node node : nodes) {
-                    nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(),
-                                                                               Node.State.active,
-                                                                               node.type(),
-                                                                               node.owner(),
-                                                                               node.currentVersion(),
-                                                                               application.version().get()));
-                }
-                serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(),
-                                                                     deployment.zoneId(),
-                                                                     false,
-                                                                     2,
-                                                                     nodes.stream()
-                                                                          .map(node -> new ServiceConvergence.Status(node.hostname(),
-                                                                                                                     43,
-                                                                                                                     "container",
-                                                                                                                     1))
-                                                                          .collect(Collectors.toList())));
-
-                PrepareResponse prepareResponse = new PrepareResponse();
-                prepareResponse.message = "foo";
-                prepareResponse.configChangeActions = configChangeActions != null
-                        ? configChangeActions
-                        : new ConfigChangeActions(Collections.emptyList(),
-                                                  Collections.emptyList());
-                setConfigChangeActions(null);
-                prepareResponse.tenant = new TenantId("tenant");
-                prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList());
-                return prepareResponse;
+        this.rotationNames.put(deployment, Set.copyOf(rotationNames));
+
+        return () -> {
+            Application application = applications.get(deployment.applicationId());
+            application.activate();
+            List<Node> nodes = nodeRepository.list(deployment.zoneId(), deployment.applicationId());
+            for (Node node : nodes) {
+                nodeRepository.putByHostname(deployment.zoneId(), new Node(node.hostname(),
+                                                                           Node.State.active,
+                                                                           node.type(),
+                                                                           node.owner(),
+                                                                           node.currentVersion(),
+                                                                           application.version().get()));
             }
-
+            serviceStatus.put(deployment, new ServiceConvergence(deployment.applicationId(),
+                                                                 deployment.zoneId(),
+                                                                 false,
+                                                                 2,
+                                                                 nodes.stream()
+                                                                      .map(node -> new ServiceConvergence.Status(node.hostname(),
+                                                                                                                 43,
+                                                                                                                 "container",
+                                                                                                                 1))
+                                                                      .collect(Collectors.toList())));
+
+            PrepareResponse prepareResponse = new PrepareResponse();
+            prepareResponse.message = "foo";
+            prepareResponse.configChangeActions = configChangeActions != null
+                    ? configChangeActions
+                    : new ConfigChangeActions(Collections.emptyList(),
+                                              Collections.emptyList());
+            setConfigChangeActions(null);
+            prepareResponse.tenant = new TenantId("tenant");
+            prepareResponse.log = warnings.getOrDefault(deployment, Collections.emptyList());
+            return prepareResponse;
         };
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
index 410d7950e97..98b2dd2f7f3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
@@ -1,14 +1,17 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 
+import java.net.URI;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 /**
  * Returns a default set of endpoints on every query if it has no mappings, or those added by the user, otherwise.
@@ -34,6 +37,14 @@ public class RoutingGeneratorMock implements RoutingGenerator {
                 : routingTable.getOrDefault(deployment, Collections.emptyList());
     }
 
+    @Override
+    public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId deployment) {
+        return endpoints(deployment).stream()
+                                    .limit(1)
+                                    .collect(Collectors.toMap(__ -> ClusterSpec.Id.from("default"),
+                                                              endpoint -> URI.create(endpoint.endpoint())));
+    }
+
     public void putEndpoints(DeploymentId deployment, List<RoutingEndpoint> endpoints) {
         routingTable.put(deployment, endpoints);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
index 5cf8268c18e..700e6e9cb42 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneFilterMock.java
@@ -43,7 +43,7 @@ public class ZoneFilterMock implements ZoneList {
 
     @Override
     public ZoneList all() {
-        return filter(zoneId -> true);
+        return filter(zone -> true);
     }
 
     @Override
@@ -63,17 +63,17 @@ public class ZoneFilterMock implements ZoneList {
 
     @Override
     public ZoneList in(Environment... environments) {
-        return filter(zoneId -> new HashSet<>(Arrays.asList(environments)).contains(zoneId.environment()));
+        return filter(zone -> new HashSet<>(Arrays.asList(environments)).contains(zone.getEnvironment()));
     }
 
     @Override
     public ZoneList in(RegionName... regions) {
-        return filter(zoneId -> new HashSet<>(Arrays.asList(regions)).contains(zoneId.region()));
+        return filter(zone -> new HashSet<>(Arrays.asList(regions)).contains(zone.getRegionName()));
     }
 
     @Override
     public ZoneList among(ZoneId... zones) {
-        return filter(zoneId -> new HashSet<>(Arrays.asList(zones)).contains(zoneId));
+        return filter(zone -> new HashSet<>(Arrays.asList(zones)).contains(zone.getId()));
     }
 
     @Override
@@ -88,15 +88,15 @@ public class ZoneFilterMock implements ZoneList {
 
     @Override
     public ZoneList ofCloud(CloudName cloud) {
-        return filter(zoneId -> zoneId.cloud().equals(cloud));
+        return filter(zone -> zone.getCloudName().equals(cloud));
     }
 
-    private ZoneFilterMock filter(Predicate<ZoneId> condition) {
+    private ZoneFilterMock filter(Predicate<ZoneApi> condition) {
         return new ZoneFilterMock(
                 zones.stream()
-                        .filter(zoneApi -> negate ?
-                                condition.negate().test(zoneApi.toDeprecatedId()) :
-                                condition.test(zoneApi.toDeprecatedId()))
+                        .filter(zone -> negate ?
+                                condition.negate().test(zone) :
+                                condition.test(zone))
                         .collect(Collectors.toList()),
                 false);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
index d94c40e6b99..449ca509ee4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
@@ -9,7 +9,9 @@ import com.yahoo.config.provision.RotationName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
@@ -18,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import org.junit.Test;
 
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -195,6 +198,20 @@ public class RoutingPoliciesTest {
         assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies().get(app1.id()).size());
     }
 
+    @Test
+    public void cluster_endpoints_resolve_from_policies() {
+        provisionLoadBalancers(3, app1.id(), zone1);
+        tester.deployCompletely(app1, applicationPackage);
+        tester.controllerTester().routingGenerator().putEndpoints(new DeploymentId(app1.id(), zone1), Collections.emptyList());
+        assertEquals(Map.of(ClusterSpec.Id.from("c0"),
+                            URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"),
+                            ClusterSpec.Id.from("c1"),
+                            URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"),
+                            ClusterSpec.Id.from("c2"),
+                            URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")),
+                     tester.controller().applications().clusterEndpoints(new DeploymentId(app1.id(), zone1)));
+    }
+
     private Set<RoutingPolicy> policies(Application application) {
         return tester.controller().curator().readRoutingPolicies(application.id());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 2c17abf9e90..71c4b41a276 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -709,14 +709,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
         // Invalid deployment fails
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation", GET)
                                       .userIdentity(USER_ID),
-                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in zone prod.us-east-3 in default\"}",
+                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}",
                               404);
 
         // Change status of non-existing deployment fails
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-east-3/global-rotation/override", PUT)
                                       .userIdentity(USER_ID)
                                       .data("{\"reason\":\"unit-test\"}"),
-                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in zone prod.us-east-3 in default\"}",
+                              "{\"error-code\":\"NOT_FOUND\",\"message\":\"application 'tenant1.application1.instance1' has no deployment in prod.us-east-3\"}",
                               404);
 
         // GET global rotation status
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
index c90d5759791..c0e9d10a40c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
@@ -223,7 +223,7 @@
   ],
   "rotationId": "rotation-id-1",
   "instances": [
-    @include(dev-us-west-1.json),
+    @include(dev-us-east-1.json),
     @include(prod-us-central-1.json)
   ],
   "metrics": {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json
index b051fb38a41..ad7e4f00027 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json
@@ -1,4 +1,4 @@
 {
   "error-code": "BAD_REQUEST",
-  "message": "Could not delete 'application 'tenant1.application1.instance1'': It has active deployments in: zone dev.us-west-1 in default, zone prod.us-central-1 in default"
+  "message": "Could not delete 'application 'tenant1.application1.instance1'': It has active deployments in: dev.us-west-1, prod.us-central-1"
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
index 1a2025e4de2..1a2025e4de2 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
new file mode 100644
index 00000000000..fef3cf6a372
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
@@ -0,0 +1,21 @@
+{
+  "application": "tenant1:application1:default",
+  "zone": "dev.us-east-1",
+  "system": "main",
+  "endpoints": {
+    "dev.us-east-1": [
+      "http://old-endpoint.vespa.yahooapis.com:4080"
+    ],
+    "prod.us-central-1": [
+      "http://old-endpoint.vespa.yahooapis.com:4080"
+    ]
+  },
+  "zoneEndpoints": {
+    "dev.us-east-1": {
+      "default": "http://old-endpoint.vespa.yahooapis.com:4080"
+    },
+    "prod.us-central-1": {
+      "default": "http://old-endpoint.vespa.yahooapis.com:4080"
+    }
+  }
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
index 77d5b3479be..390024fe33d 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/us-east-3-log-without-first.json
@@ -12,7 +12,7 @@
       {
         "at": 1000,
         "type": "debug",
-        "message": "Deactivating tester of tenant.application in zone prod.us-east-3 in default ..."
+        "message": "Deactivating tester of tenant.application in prod.us-east-3 ..."
       }
     ]
   },
diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json
new file mode 100644
index 00000000000..4145d863995
--- /dev/null
+++ b/controller-server/src/test/resources/testConfig.json
@@ -0,0 +1,20 @@
+{
+  "application": "tenant:application:default",
+  "zone": "test.aws-us-east-1c",
+  "system": "publiccd",
+  "endpoints": {
+    "test.aws-us-east-1c": [
+      "https://server/"
+    ]
+  },
+  "zoneEndpoints": {
+    "test.aws-us-east-1c": {
+      "ai": "https://server/"
+    }
+  },
+  "clusters": {
+    "test.aws-us-east-1c": [
+      "facts"
+    ]
+  }
+}