Merge pull request #9759 from vespa-engine/jvenstad/tenant-cd

Jvenstad/tenant cd

16 files changed, 290 insertions, 94 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index e0a73d994d1..a079c8df78e 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -7,6 +7,7 @@ import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.zone.ZoneId;
@@ -76,6 +77,7 @@ import java.security.Principal;
 import java.time.Clock;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.EnumSet;
@@ -86,7 +88,9 @@ import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
+import java.util.TreeMap;
 import java.util.function.Consumer;
+import java.util.function.Predicate;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -520,26 +524,60 @@ public class ApplicationController {
         controller.nameServiceForwarder().createCname(RecordName.from(name), RecordData.fqdn(targetName), Priority.normal);
     }
 
-    /** Returns the endpoints of the deployment, or an empty list if the request fails */
-    public Optional<List<URI>> getDeploymentEndpoints(DeploymentId deploymentId) {
+    /** Returns the endpoints of the deployment, or empty if the request fails */
+    public List<URI> getDeploymentEndpoints(DeploymentId deploymentId) {
         if ( ! get(deploymentId.applicationId())
                 .map(application -> application.deployments().containsKey(deploymentId.zoneId()))
                 .orElse(deploymentId.applicationId().instance().isTester()))
             throw new NotExistsException("Deployment", deploymentId.toString());
 
         try {
-            return Optional.of(ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
-                                                                    .map(RoutingEndpoint::endpoint)
-                                                                    .map(URI::create)
-                                                                    .iterator()));
+            return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
+                                                        .map(RoutingEndpoint::endpoint)
+                                                        .map(URI::create)
+                                                        .iterator());
         }
         catch (RuntimeException e) {
             log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId + ": "
                                    + Exceptions.toMessageString(e));
-            return Optional.empty();
+            return Collections.emptyList();
         }
     }
 
+    /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */
+    public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) {
+        if ( ! get(id.applicationId())
+                .map(application -> application.deployments().containsKey(id.zoneId()))
+                .orElse(id.applicationId().instance().isTester()))
+            throw new NotExistsException("Deployment", id.toString());
+
+        try {
+            return Optional.of(routingPolicies.get(id))
+                           .filter(policies -> ! policies.isEmpty())
+                           .map(policies -> policies.stream()
+                                                    .filter(policy -> policy.endpointIn(controller.system()).scope() == Endpoint.Scope.zone)
+                                                    .collect(Collectors.toUnmodifiableMap(policy -> policy.cluster(),
+                                                                                          policy -> policy.endpointIn(controller.system()).url())))
+                           .orElseGet(() -> routingGenerator.clusterEndpoints(id));
+        }
+        catch (RuntimeException e) {
+            log.log(Level.WARNING, "Failed to get endpoint information for " + id + ": "
+                                   + Exceptions.toMessageString(e));
+            return Collections.emptyMap();
+        }
+    }
+
+    /** Returns all zone-specific cluster endpoints for the given application, in the given zones. */
+    public Map<ZoneId, Map<ClusterSpec.Id, URI>> clusterEndpoints(ApplicationId id, Collection<ZoneId> zones) {
+        Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments = new TreeMap<>(Comparator.comparing(ZoneId::value));
+        for (ZoneId zone : zones) {
+            var endpoints = clusterEndpoints(new DeploymentId(id, zone));
+            if ( ! endpoints.isEmpty())
+                deployments.put(zone, endpoints);
+        }
+        return Collections.unmodifiableMap(deployments);
+    }
+
     /**
      * Deletes the the given application. All known instances of the applications will be deleted,
      * including PR instances.
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
index 891a696be9c..bf2460284ab 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunner.java
@@ -11,13 +11,11 @@ import com.yahoo.config.application.api.Notifications.When;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.io.IOUtils;
 import com.yahoo.log.LogLevel;
-import com.yahoo.slime.Cursor;
-import com.yahoo.slime.Slime;
-import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.ActivateResult;
@@ -28,7 +26,6 @@ import com.yahoo.vespa.hosted.controller.api.integration.LogEntry;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.Node;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.PrepareResponse;
-import com.yahoo.vespa.hosted.controller.api.integration.configserver.ServiceConvergence;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.RunId;
@@ -41,7 +38,6 @@ import com.yahoo.vespa.hosted.controller.application.DeploymentJobs.JobReport;
 import com.yahoo.yolean.Exceptions;
 
 import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.io.PrintStream;
 import java.io.UncheckedIOException;
 import java.net.URI;
@@ -57,7 +53,6 @@ import java.util.function.Supplier;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import static com.yahoo.config.application.api.Notifications.Role.author;
 import static com.yahoo.config.application.api.Notifications.When.failing;
@@ -99,10 +94,12 @@ public class InternalStepRunner implements StepRunner {
     static final Duration installationTimeout = Duration.ofMinutes(150);
 
     private final Controller controller;
+    private final TestConfigSerializer testConfigSerializer;
     private final DeploymentFailureMails mails;
 
     public InternalStepRunner(Controller controller) {
         this.controller = controller;
+        this.testConfigSerializer = new TestConfigSerializer(controller.system());
         this.mails = new DeploymentFailureMails(controller.zoneRegistry());
     }
 
@@ -320,19 +317,23 @@ public class InternalStepRunner implements StepRunner {
 
     private boolean endpointsAvailable(ApplicationId id, ZoneId zoneId, DualLogger logger) {
         logger.log("Attempting to find deployment endpoints ...");
-        Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id, Set.of(zoneId));
+        var endpoints = controller.applications().clusterEndpoints(id, Set.of(zoneId));
         if ( ! endpoints.containsKey(zoneId)) {
             logger.log("Endpoints not yet ready.");
             return false;
         }
+        logEndpoints(endpoints, logger);
+        return true;
+    }
+
+    private void logEndpoints(Map<ZoneId, Map<ClusterSpec.Id, URI>> endpoints, DualLogger logger) {
         List<String> messages = new ArrayList<>();
         messages.add("Found endpoints:");
         endpoints.forEach((zone, uris) -> {
             messages.add("- " + zone);
-            uris.forEach(uri -> messages.add(" |-- " + uri));
+            uris.forEach((cluster, uri) -> messages.add(" |-- " + uri + " (" + cluster + ")"));
         });
         logger.log(messages);
-        return true;
     }
 
     private boolean nodesConverged(ApplicationId id, JobType type, Version target, DualLogger logger) {
@@ -384,21 +385,15 @@ public class InternalStepRunner implements StepRunner {
             return Optional.of(aborted);
         }
 
-        Set<ZoneId> zones = testedZoneAndProductionZones(id);
+        Set<ZoneId> zones = controller.jobController().testedZoneAndProductionZones(id.application(), id.type());
 
         logger.log("Attempting to find endpoints ...");
-        Map<ZoneId, List<URI>> endpoints = deploymentEndpoints(id.application(), zones);
+        var endpoints = controller.applications().clusterEndpoints(id.application(), zones);
         if ( ! endpoints.containsKey(id.type().zone(controller.system())) && timedOut(deployment.get(), endpointTimeout)) {
             logger.log(WARNING, "Endpoints for the deployment to test vanished again, while it was still active!");
             return Optional.of(error);
         }
-        List<String> messages = new ArrayList<>();
-        messages.add("Found endpoints:");
-        endpoints.forEach((zone, uris) -> {
-            messages.add("- " + zone);
-            uris.forEach(uri -> messages.add(" |-- " + uri));
-        });
-        logger.log(messages);
+        logEndpoints(endpoints, logger);
 
         Optional<URI> testerEndpoint = controller.jobController().testerEndpoint(id);
         if (testerEndpoint.isEmpty() && timedOut(deployment.get(), endpointTimeout)) {
@@ -410,9 +405,10 @@ public class InternalStepRunner implements StepRunner {
             logger.log("Starting tests ...");
             controller.jobController().cloud().startTests(testerEndpoint.get(),
                                                           TesterCloud.Suite.of(id.type()),
-                                                          testConfig(id.application(), id.type().zone(controller.system()),
-                                                                     controller.system(), endpoints,
-                                                                     listClusters(id.application(), zones)));
+                                                          testConfigSerializer.configJson(id.application(),
+                                                                                          id.type(),
+                                                                                          endpoints,
+                                                                                          listClusters(id.application(), zones)));
             return Optional.of(running);
         }
 
@@ -612,28 +608,6 @@ public class InternalStepRunner implements StepRunner {
         throw new IllegalStateException("No step deploys to the zone this run is for!");
     }
 
-    /** Returns a stream containing the zone of the deployment tested in the given run, and all production zones for the application. */
-    private Set<ZoneId> testedZoneAndProductionZones(RunId id) {
-        return Stream.concat(Stream.of(id.type().zone(controller.system())),
-                             application(id.application()).productionDeployments().keySet().stream())
-                     .collect(Collectors.toSet());
-    }
-
-    /** Returns all endpoints for all current deployments of the given real application. */
-    private Map<ZoneId, List<URI>> deploymentEndpoints(ApplicationId id, Iterable<ZoneId> zones) {
-        ImmutableMap.Builder<ZoneId, List<URI>> deployments = ImmutableMap.builder();
-        for (ZoneId zone : zones) {
-            controller.applications().getDeploymentEndpoints(new DeploymentId(id, zone))
-                      .filter(endpoints -> ! endpoints.isEmpty())
-                      .or(() -> Optional.of(controller.applications().routingPolicies().get(id, zone).stream()
-                                                      .map(policy -> policy.endpointIn(controller.system()).url())
-                                                      .collect(Collectors.toUnmodifiableList()))
-                                        .filter(endpoints -> ! endpoints.isEmpty()))
-                      .ifPresent(endpoints -> deployments.put(zone, endpoints));
-        }
-        return deployments.build();
-    }
-
     /** Returns all content clusters in all current deployments of the given real application. */
     private Map<ZoneId, List<String>> listClusters(ApplicationId id, Iterable<ZoneId> zones) {
         ImmutableMap.Builder<ZoneId, List<String>> clusters = ImmutableMap.builder();
@@ -712,38 +686,6 @@ public class InternalStepRunner implements StepRunner {
         return deploymentSpec.getBytes(StandardCharsets.UTF_8);
     }
 
-    /** Returns the config for the tests to run for the given job. */
-    private static byte[] testConfig(ApplicationId id, ZoneId testerZone, SystemName system,
-                                     Map<ZoneId, List<URI>> deployments, Map<ZoneId, List<String>> clusters) {
-        Slime slime = new Slime();
-        Cursor root = slime.setObject();
-
-        root.setString("application", id.serializedForm());
-        root.setString("zone", testerZone.value());
-        root.setString("system", system.value());
-
-        Cursor endpointsObject = root.setObject("endpoints");
-        deployments.forEach((zone, endpoints) -> {
-            Cursor endpointArray = endpointsObject.setArray(zone.value());
-            for (URI endpoint : endpoints)
-                endpointArray.addString(endpoint.toString());
-        });
-
-        Cursor clustersObject = root.setObject("clusters");
-        clusters.forEach((zone, clusterList) -> {
-            Cursor clusterArray = clustersObject.setArray(zone.value());
-            for (String cluster : clusterList)
-                clusterArray.addString(cluster);
-        });
-
-        try {
-            return SlimeUtils.toJsonBytes(slime);
-        }
-        catch (IOException e) {
-            throw new UncheckedIOException(e);
-        }
-    }
-
     /** Logger which logs to a {@link JobController}, as well as to the parent class' {@link Logger}. */
     private class DualLogger {
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 5bb9686ce0f..c644af2e554 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 import com.google.common.collect.ImmutableMap;
 import com.yahoo.component.Version;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Application;
 import com.yahoo.vespa.hosted.controller.Controller;
@@ -41,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.UnaryOperator;
 import java.util.logging.Level;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 import static com.google.common.collect.ImmutableList.copyOf;
@@ -408,12 +410,19 @@ public class JobController {
     Optional<URI> testerEndpoint(RunId id) {
         DeploymentId testerId = new DeploymentId(id.tester().id(), id.type().zone(controller.system()));
         return controller.applications().getDeploymentEndpoints(testerId)
-                         .flatMap(uris -> uris.stream().findAny())
+                         .stream().findAny()
                          .or(() -> controller.applications().routingPolicies().get(testerId).stream()
                                              .findAny()
                                              .map(policy -> policy.endpointIn(controller.system()).url()));
     }
 
+    /** Returns a set containing the zone of the deployment tested in the given run, and all production zones for the application. */
+    public Set<ZoneId> testedZoneAndProductionZones(ApplicationId id, JobType type) {
+        return Stream.concat(Stream.of(type.zone(controller.system())),
+                             controller.applications().require(id).productionDeployments().keySet().stream())
+                     .collect(Collectors.toSet());
+    }
+
     // TODO jvenstad: Find a more appropriate way of doing this, at least when this is the only build service.
     private long nextBuild(ApplicationId id) {
         return 1 + controller.applications().require(id).deploymentJobs()
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
new file mode 100644
index 00000000000..e79692d34ed
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializer.java
@@ -0,0 +1,82 @@
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.slime.Cursor;
+import com.yahoo.slime.Slime;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Serializes config for integration tests against Vespa deployments.
+ *
+ * @author jonmv
+ */
+public class TestConfigSerializer {
+
+    private final SystemName system;
+
+    public TestConfigSerializer(SystemName system) {
+        this.system = system;
+    }
+
+    public Slime configSlime(ApplicationId id,
+                             JobType type,
+                             Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments,
+                             Map<ZoneId, List<String>> clusters) {
+        Slime slime = new Slime();
+        Cursor root = slime.setObject();
+
+        root.setString("application", id.serializedForm());
+        root.setString("zone", type.zone(system).value());
+        root.setString("system", system.value());
+
+        Cursor endpointsObject = root.setObject("endpoints"); // TODO jvenstad: remove.
+        deployments.forEach((zone, endpoints) -> {
+            Cursor endpointArray = endpointsObject.setArray(zone.value());
+            for (URI endpoint : endpoints.values())
+                endpointArray.addString(endpoint.toString());
+        });
+
+        Cursor zoneEndpointsObject = root.setObject("zoneEndpoints");
+        deployments.forEach((zone, endpoints) -> {
+            Cursor clusterEndpointsObject = zoneEndpointsObject.setObject(zone.value());
+            endpoints.forEach((cluster, endpoint) -> {
+                clusterEndpointsObject.setString(cluster.value(), endpoint.toString());
+            });
+        });
+
+        if ( ! clusters.isEmpty()) {
+            Cursor clustersObject = root.setObject("clusters");
+            clusters.forEach((zone, clusterList) -> {
+                Cursor clusterArray = clustersObject.setArray(zone.value());
+                for (String cluster : clusterList)
+                    clusterArray.addString(cluster);
+            });
+        }
+
+        return slime;
+    }
+
+    /** Returns the config for the tests to run for the given job. */
+    public byte[] configJson(ApplicationId id,
+                             JobType type,
+                             Map<ZoneId, Map<ClusterSpec.Id, URI>> deployments,
+                             Map<ZoneId, List<String>> clusters) {
+        try {
+            return SlimeUtils.toJsonBytes(configSlime(id, type, deployments, clusters));
+        }
+        catch (IOException e) {
+            throw new UncheckedIOException(e);
+        }
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 78148bf5428..4402a6267fe 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -66,6 +66,7 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTrigger.ChangesToCancel;
+import com.yahoo.vespa.hosted.controller.deployment.TestConfigSerializer;
 import com.yahoo.vespa.hosted.controller.restapi.ErrorResponse;
 import com.yahoo.vespa.hosted.controller.restapi.MessageResponse;
 import com.yahoo.vespa.hosted.controller.restapi.ResourceResponse;
@@ -95,6 +96,7 @@ import java.time.Duration;
 import java.time.Instant;
 import java.util.Arrays;
 import java.util.Base64;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -182,6 +184,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/logs")) return logs(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request.propertyMap());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job")) return JobControllerApiHandlerHelper.jobTypeResponse(controller, appIdFromPath(path), request.getUri());
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}")) return JobControllerApiHandlerHelper.runResponse(controller.jobController().runs(appIdFromPath(path), jobTypeFromPath(path)), request.getUri());
+        if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/test-config")) return testConfig(appIdFromPath(path), jobTypeFromPath(path));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/instance/{instance}/job/{jobtype}/run/{number}")) return JobControllerApiHandlerHelper.runDetailsResponse(controller.jobController(), runIdFromPath(path), request.getProperty("after"));
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}")) return deployment(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
         if (path.matches("/application/v4/tenant/{tenant}/application/{application}/environment/{environment}/region/{region}/instance/{instance}/suspended")) return suspended(path.get("tenant"), path.get("application"), path.get("instance"), path.get("environment"), path.get("region"), request);
@@ -599,7 +602,7 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         // ask the routing layer here
         Cursor serviceUrlArray = response.setArray("serviceUrls");
         controller.applications().getDeploymentEndpoints(deploymentId)
-                  .ifPresent(endpoints -> endpoints.forEach(endpoint -> serviceUrlArray.addString(endpoint.toString())));
+                  .forEach(endpoint -> serviceUrlArray.addString(endpoint.toString()));
 
         response.setString("nodes", withPath("/zone/v2/" + deploymentId.zoneId().environment() + "/" + deploymentId.zoneId().region() + "/nodes/v2/node/?&recursive=true&application=" + deploymentId.applicationId().tenant() + "." + deploymentId.applicationId().application() + "." + deploymentId.applicationId().instance(), request.getUri()).toString());
         response.setString("yamasUrl", monitoringSystemUri(deploymentId).toString());
@@ -1109,6 +1112,14 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
         }
     }
 
+    private HttpResponse testConfig(ApplicationId id, JobType type) {
+        var endpoints = controller.applications().clusterEndpoints(id, controller.jobController().testedZoneAndProductionZones(id, type));
+        return new SlimeJsonResponse(new TestConfigSerializer(controller.system()).configSlime(id,
+                                                                                               type,
+                                                                                               endpoints,
+                                                                                               Collections.emptyMap()));
+    }
+
     private static DeploymentJobs.JobReport toJobReport(String tenantName, String applicationName, Inspector report) {
         Optional<DeploymentJobs.JobError> jobError = Optional.empty();
         if (report.field("jobError").valid()) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
index 370fd03d9e7..4fb6a2e34fb 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/InternalStepRunnerTest.java
@@ -3,12 +3,15 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Inspector;
+import com.yahoo.slime.JsonFormat;
+import com.yahoo.slime.ObjectTraverser;
 import com.yahoo.vespa.config.SlimeUtils;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.ConfigChangeActions;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.configserverbindings.RefeedAction;
@@ -25,6 +28,7 @@ import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.net.URI;
@@ -34,6 +38,7 @@ import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.Collections;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.Executors;
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
new file mode 100644
index 00000000000..bc411d4377d
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/TestConfigSerializerTest.java
@@ -0,0 +1,36 @@
+package com.yahoo.vespa.hosted.controller.deployment;
+
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+import java.util.Map;
+
+import static com.yahoo.vespa.hosted.controller.deployment.InternalDeploymentTester.appId;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author jonmv
+ */
+public class TestConfigSerializerTest {
+
+    @Test
+    public void testConfig() throws IOException {
+        ZoneId zone = JobType.systemTest.zone(SystemName.PublicCd);
+        byte[] json = new TestConfigSerializer(SystemName.PublicCd).configJson(appId,
+                                                                               JobType.systemTest,
+                                                                               Map.of(zone, Map.of(ClusterSpec.Id.from("ai"),
+                                                                                                   URI.create("https://server/"))),
+                                                                               Map.of(zone, List.of("facts")));
+        byte[] expected = InternalStepRunnerTest.class.getResourceAsStream("/testConfig.json").readAllBytes();
+        assertEquals(new String(SlimeUtils.toJsonBytes(SlimeUtils.jsonToSlime(expected))),
+                     new String(json));
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
index 410d7950e97..98b2dd2f7f3 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/RoutingGeneratorMock.java
@@ -1,14 +1,17 @@
 // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.integration;
 
+import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingEndpoint;
 import com.yahoo.vespa.hosted.controller.api.integration.routing.RoutingGenerator;
 
+import java.net.URI;
 import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
 
 /**
  * Returns a default set of endpoints on every query if it has no mappings, or those added by the user, otherwise.
@@ -34,6 +37,14 @@ public class RoutingGeneratorMock implements RoutingGenerator {
                 : routingTable.getOrDefault(deployment, Collections.emptyList());
     }
 
+    @Override
+    public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId deployment) {
+        return endpoints(deployment).stream()
+                                    .limit(1)
+                                    .collect(Collectors.toMap(__ -> ClusterSpec.Id.from("default"),
+                                                              endpoint -> URI.create(endpoint.endpoint())));
+    }
+
     public void putEndpoints(DeploymentId deployment, List<RoutingEndpoint> endpoints) {
         routingTable.put(deployment, endpoints);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
index d94c40e6b99..f19c309dd67 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPoliciesTest.java
@@ -9,7 +9,9 @@ import com.yahoo.config.provision.RotationName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
@@ -18,6 +20,7 @@ import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import org.junit.Test;
 
+import java.net.URI;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
@@ -195,6 +198,19 @@ public class RoutingPoliciesTest {
         assertEquals("Keeps routing policies for " + app1, 4, tester.controller().applications().routingPolicies().get(app1.id()).size());
     }
 
+    @Test
+    public void cluster_endpoints_resolve_from_policies() {
+        provisionLoadBalancers(3, app1.id(), zone1);
+        tester.deployCompletely(app1, applicationPackage);
+        assertEquals(Map.of(ClusterSpec.Id.from("c0"),
+                            URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"),
+                            ClusterSpec.Id.from("c1"),
+                            URI.create("https://c1.app1.tenant1.us-west-1.vespa.oath.cloud/"),
+                            ClusterSpec.Id.from("c2"),
+                            URI.create("https://c2.app1.tenant1.us-west-1.vespa.oath.cloud/")),
+                     tester.controller().applications().clusterEndpoints(new DeploymentId(app1.id(), zone1)));
+    }
+
     private Set<RoutingPolicy> policies(Application application) {
         return tester.controller().curator().readRoutingPolicies(application.id());
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index 99bc07a9528..632fea8cb1f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -209,7 +209,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
 
         // POST (deploy) an application to a zone - manual user deployment (includes a content hash for verification)
         MultiPartStreamer entity = createApplicationDeployData(applicationPackage, true);
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default/deploy", POST)
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default/deploy", POST)
                                       .data(entity)
                                       .header("X-Content-Hash", Base64.getEncoder().encodeToString(Signatures.sha256Digest(entity::data)))
                                       .userIdentity(USER_ID),
@@ -521,10 +521,15 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .oktaAccessToken(OKTA_AT),
                               new File("delete-with-active-deployments.json"), 400);
 
+        // GET config for running a test against a deployment
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/default/job/dev-us-east-1/test-config", GET)
+                                      .userIdentity(USER_ID),
+                              new File("test-config.json"));
+
         // DELETE (deactivate) a deployment - dev
-        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default", DELETE)
+        tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default", DELETE)
                                       .userIdentity(USER_ID),
-                              "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default");
+                              "Deactivated tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default");
 
         // DELETE (deactivate) a deployment - prod
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/environment/prod/region/us-central-1/instance/default", DELETE)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
index 4b2cb397b5b..ce29165d497 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application.json
@@ -225,9 +225,9 @@
   "instances": [
     {
       "environment": "dev",
-      "region": "us-west-1",
+      "region": "us-east-1",
       "instance": "default",
-      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-west-1/instance/default"
+      "url": "http://localhost:8080/application/v4/tenant/tenant1/application/application1/environment/dev/region/us-east-1/instance/default"
     },
     {
       "bcpStatus": {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
index fa903b61825..de99019447a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/application1-recursive.json
@@ -223,7 +223,7 @@
   ],
   "rotationId": "rotation-id-1",
   "instances": [
-    @include(dev-us-west-1.json),
+    @include(dev-us-east-1.json),
     @include(prod-us-central-1.json)
   ],
   "metrics": {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json
index 2b17d55627a..012b0031489 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/delete-with-active-deployments.json
@@ -1,4 +1,4 @@
 {
   "error-code": "BAD_REQUEST",
-  "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-west-1 in default, zone prod.us-central-1 in default"
+  "message": "Could not delete 'application 'tenant1.application1'': It has active deployments in: zone dev.us-east-1 in default, zone prod.us-central-1 in default"
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
index 67c71ee3880..7cbbe4be43b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-west-1.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/dev-us-east-1.json
@@ -3,7 +3,7 @@
   "application": "application1",
   "instance": "default",
   "environment": "dev",
-  "region": "us-west-1",
+  "region": "us-east-1",
   "endpoints": [],
   "serviceUrls": [
     "http://old-endpoint.vespa.yahooapis.com:4080",
@@ -12,8 +12,8 @@
     "http://global-endpoint.vespa.yahooapis.com:4080",
     "http://alias-endpoint.vespa.yahooapis.com:4080"
   ],
-  "nodes": "http://localhost:8080/zone/v2/dev/us-west-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default",
-  "yamasUrl": "http://monitoring-system.test/?environment=dev&region=us-west-1&application=tenant1.application1",
+  "nodes": "http://localhost:8080/zone/v2/dev/us-east-1/nodes/v2/node/%3F&recursive=true&application=tenant1.application1.default",
+  "yamasUrl": "http://monitoring-system.test/?environment=dev&region=us-east-1&application=tenant1.application1",
   "version": "(ignore)",
   "revision": "(ignore)",
   "deployTimeEpochMs": "(ignore)",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
new file mode 100644
index 00000000000..fef3cf6a372
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/responses/test-config.json
@@ -0,0 +1,21 @@
+{
+  "application": "tenant1:application1:default",
+  "zone": "dev.us-east-1",
+  "system": "main",
+  "endpoints": {
+    "dev.us-east-1": [
+      "http://old-endpoint.vespa.yahooapis.com:4080"
+    ],
+    "prod.us-central-1": [
+      "http://old-endpoint.vespa.yahooapis.com:4080"
+    ]
+  },
+  "zoneEndpoints": {
+    "dev.us-east-1": {
+      "default": "http://old-endpoint.vespa.yahooapis.com:4080"
+    },
+    "prod.us-central-1": {
+      "default": "http://old-endpoint.vespa.yahooapis.com:4080"
+    }
+  }
+}
diff --git a/controller-server/src/test/resources/testConfig.json b/controller-server/src/test/resources/testConfig.json
new file mode 100644
index 00000000000..4145d863995
--- /dev/null
+++ b/controller-server/src/test/resources/testConfig.json
@@ -0,0 +1,20 @@
+{
+  "application": "tenant:application:default",
+  "zone": "test.aws-us-east-1c",
+  "system": "publiccd",
+  "endpoints": {
+    "test.aws-us-east-1c": [
+      "https://server/"
+    ]
+  },
+  "zoneEndpoints": {
+    "test.aws-us-east-1c": {
+      "ai": "https://server/"
+    }
+  },
+  "clusters": {
+    "test.aws-us-east-1c": [
+      "facts"
+    ]
+  }
+}