Merge pull request #11983 from vespa-engine/mpolden/status-in-directly-routed-zone

Allow status change in zones that are exclusively directly routed

15 files changed, 194 insertions, 88 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
index c44c533e995..f28c877dce6 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/ApplicationController.java
@@ -251,7 +251,6 @@ public class ApplicationController {
     /** Change status of all global endpoints for given deployment */
     public void setGlobalRotationStatus(DeploymentId deployment, EndpointStatus status) {
         var globalEndpoints = findGlobalEndpoints(deployment);
-        if (globalEndpoints.isEmpty()) throw new IllegalArgumentException(deployment + " has no global endpoints");
         globalEndpoints.forEach(endpoint -> {
             try {
                 configServer.setGlobalRotationStatus(deployment, endpoint.upstreamName(), status);
@@ -620,10 +619,10 @@ public class ApplicationController {
             throw new NotExistsException("Deployment", deploymentId.toString());
 
         try {
-            return ImmutableList.copyOf(routingGenerator.endpoints(deploymentId).stream()
-                                                        .map(RoutingEndpoint::endpoint)
-                                                        .map(URI::create)
-                                                        .iterator());
+            return findRoutingEndpoints(deploymentId).stream()
+                                                     .map(RoutingEndpoint::endpoint)
+                                                     .map(URI::create)
+                                                     .collect(Collectors.toUnmodifiableList());
         }
         catch (RuntimeException e) {
             log.log(Level.WARNING, "Failed to get endpoint information for " + deploymentId, e);
@@ -631,6 +630,21 @@ public class ApplicationController {
         }
     }
 
+    /** Find the routing endpoints of a given deployment. Routing endpoints are owned by the shared routing layer. */
+    private List<RoutingEndpoint> findRoutingEndpoints(DeploymentId deployment) {
+        if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) {
+            return List.of(); // No shared routing layer in this zone.
+        }
+        return routingGenerator.endpoints(deployment);
+    }
+
+    private Map<ClusterSpec.Id, URI> findClusterEndpoints(DeploymentId deployment) {
+        if (controller.zoneRegistry().zones().directlyRouted().ids().contains(deployment.zoneId())) {
+            return Map.of(); // No shared routing layer in this zone.
+        }
+        return routingGenerator.clusterEndpoints(deployment);
+    }
+
     /** Returns the non-empty endpoints per cluster in the given deployment, or empty if endpoints can't be found. */
     public Map<ClusterSpec.Id, URI> clusterEndpoints(DeploymentId id) {
         if ( ! getInstance(id.applicationId())
@@ -640,7 +654,7 @@ public class ApplicationController {
 
         // TODO(jvenstad): Swap to use routingPolicies first, when this is ready.
         try {
-            var endpoints = routingGenerator.clusterEndpoints(id);
+            var endpoints = findClusterEndpoints(id);
             if ( ! endpoints.isEmpty())
                 return endpoints;
         }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
index 6d7980396de..13b3b368293 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/deployment/JobController.java
@@ -3,6 +3,7 @@ package com.yahoo.vespa.hosted.controller.deployment;
 
 import com.google.common.collect.ImmutableSortedMap;
 import com.yahoo.component.Version;
+import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
@@ -506,12 +507,17 @@ public class JobController {
                });
     }
 
+    // TODO(mpolden): Eliminate duplication in this and ApplicationController#deactivate
     public void deactivateTester(TesterId id, JobType type) {
+        var zone = type.zone(controller.system());
         try {
-            controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), type.zone(controller.system())));
-        }
-        catch (NotFoundException ignored) {
+            controller.serviceRegistry().configServer().deactivate(new DeploymentId(id.id(), zone));
+        } catch (NotFoundException ignored) {
             // Already gone -- great!
+        } finally {
+            // Passing an empty DeploymentSpec here is fine as it's used for registering global endpoint names, and
+            // tester instances have none.
+            controller.applications().routingPolicies().refresh(id.id(), DeploymentSpec.empty, zone);
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
index 4c7305f08e5..ad8c0c73754 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManager.java
@@ -30,6 +30,9 @@ import java.util.logging.Logger;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
+/**
+ * @author andreer
+ */
 public class EndpointCertificateManager {
 
     private static final Logger log = Logger.getLogger(EndpointCertificateManager.class.getName());
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index cad206b0a69..be459614880 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -83,8 +83,8 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         } else {
             controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in);
         }
-        return new MessageResponse("Set global routing status for deployments in " + zone + " to '" +
-                                   (in ? "in" : "out") + "'");
+        return new MessageResponse("Set global routing status for deployments in " + zone + " to " +
+                                   (in ? "IN" : "OUT"));
     }
 
     private HttpResponse zoneStatus(Path path) {
@@ -120,7 +120,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
 
         // Set policy status
         controller.applications().routingPolicies().setGlobalRoutingStatus(deployment, status, agent);
-        return new MessageResponse("Set global routing status for " + deployment + " to '" + (in ? "in" : "out") + "'");
+        return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT"));
     }
 
     private HttpResponse deploymentStatus(Path path) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index 88f746a9c51..961a3674471 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -84,10 +84,10 @@ public class RoutingPolicies {
                                                        deploymentSpec);
         var inactiveZones = inactiveZones(application, deploymentSpec);
         try (var lock = db.lockRoutingPolicies()) {
-            removeGlobalDnsUnreferencedBy(loadBalancers, lock);
+            if (!application.instance().isTester()) removeGlobalDnsUnreferencedBy(loadBalancers, lock);
             storePoliciesOf(loadBalancers, lock);
             removePoliciesUnreferencedBy(loadBalancers, lock);
-            updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock);
+            if (!application.instance().isTester()) updateGlobalDnsOf(get(loadBalancers.application).values(), inactiveZones, lock);
         }
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index 02c86f78ec0..83f80488ee5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -210,17 +210,6 @@ public class ControllerTest {
         assertEquals(2, upstreamOneEndpoints.get().size());
         assertTrue("All upstreams are out", upstreamOneEndpoints.get().stream().allMatch(es -> es.getStatus() == EndpointStatus.Status.out));
         assertTrue("Reason is set", upstreamOneEndpoints.get().stream().allMatch(es -> es.getReason().equals("unit-test")));
-
-        // Deployment without a global endpoint
-        tester.serviceRegistry().routingGeneratorMock().putEndpoints(deployment, List.of(
-                new RoutingEndpoint("http://old-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream2"),
-                new RoutingEndpoint("http://qrs-endpoint.vespa.yahooapis.com:4080", "host1", false, "upstream1"),
-                new RoutingEndpoint("http://feeding-endpoint.vespa.yahooapis.com:4080", "host2", false, "upstream3")
-        ));
-        try {
-            tester.controller().applications().setGlobalRotationStatus(deployment, status);
-            fail("Expected exception");
-        } catch (IllegalArgumentException ignored) {}
     }
 
     @Test
@@ -715,8 +704,10 @@ public class ControllerTest {
 
         // Create app1
         var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
-        var applicationPackage = new ApplicationPackageBuilder().environment(Environment.prod)
-                                                                .region("us-west-1")
+        var prodZone = ZoneId.from("prod", "us-west-1");
+        tester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(prodZone));
+        var applicationPackage = new ApplicationPackageBuilder().environment(prodZone.environment())
+                                                                .region(prodZone.region())
                                                                 .build();
         // Deploy app1 in production
         context1.submit(applicationPackage).deploy();
@@ -725,7 +716,7 @@ public class ControllerTest {
         assertEquals(Stream.concat(Stream.of("vznqtz7a5ygwjkbhhj7ymxvlrekgt4l6g.vespa.oath.cloud",
                                              "app1.tenant1.global.vespa.oath.cloud",
                                              "*.app1.tenant1.global.vespa.oath.cloud"),
-                                   tester.controller().zoneRegistry().zones().all().ids().stream()
+                                   tester.controller().zoneRegistry().zones().directlyRouted().ids().stream()
                                          .flatMap(zone -> Stream.of("", "*.")
                                                                 .map(prefix -> prefix + "app1.tenant1." + zone.region().value() +
                                                                                (zone.environment() == Environment.prod ? "" :  "." + zone.environment().value()) +
@@ -739,13 +730,13 @@ public class ControllerTest {
 
         // Create app2
         var context2 = tester.newDeploymentContext("tenant1", "app2", "default");
-        ZoneId zone = ZoneId.from("dev", "us-east-1");
+        var devZone = ZoneId.from("dev", "us-east-1");
 
         // Deploy app2, after "removing" direct routing everywhere
         tester.controllerTester().zoneRegistry().setDirectlyRouted();
-        tester.controller().applications().deploy(context2.instanceId(), zone, Optional.of(applicationPackage), DeployOptions.none());
+        tester.controller().applications().deploy(context2.instanceId(), devZone, Optional.of(applicationPackage), DeployOptions.none());
         assertTrue("Application deployed and activated",
-                   tester.configServer().application(context2.instanceId(), zone).get().activated());
+                   tester.configServer().application(context2.instanceId(), devZone).get().activated());
         assertFalse("Does not provision certificate in zones with routing layer", certificate.apply(context2.instance()).isPresent());
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index ebf64f90873..c90abe5ed5c 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.security.KeyAlgorithm;
@@ -17,6 +18,7 @@ import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.ConfigServerException;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.ApplicationVersion;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobId;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
@@ -99,6 +101,7 @@ public class DeploymentContext {
     private final RoutingGeneratorMock routing;
     private final JobRunner runner;
     private final DeploymentTester tester;
+    private final Set<Environment> deferLoadBalancerProvisioning = new HashSet<>();
 
     private ApplicationVersion lastSubmission = null;
     private boolean deferDnsUpdates = false;
@@ -190,6 +193,15 @@ public class DeploymentContext {
         return this;
     }
 
+    /**
+     * Defer provisioning of load balancers in zones in given environment. Default behaviour is to automatically
+     * provision load balancers in all zones that support direct routing.
+     */
+    public DeploymentContext deferLoadBalancerProvisioningIn(Environment... environment) {
+        deferLoadBalancerProvisioning.addAll(List.of(environment));
+        return this;
+    }
+
 
     /** Defer DNS updates */
     public DeploymentContext deferDnsUpdates() {
@@ -438,6 +450,25 @@ public class DeploymentContext {
         ZoneId zone = zone(job);
         DeploymentId deployment = new DeploymentId(job.application(), zone);
 
+        // Provision load balancers in directly routed zones, unless explicitly deferred
+        if (provisionLoadBalancerIn(zone)) {
+            if (job.type().isTest()) {
+                var testerDeployment = new DeploymentId(testerId.id(), zone);
+                configServer().putLoadBalancers(zone, List.of(new LoadBalancer(testerDeployment.toString(),
+                                                                               testerDeployment.applicationId(),
+                                                                               ClusterSpec.Id.from("default"),
+                                                                               HostName.from("lb-host"),
+                                                                               LoadBalancer.State.active,
+                                                                               Optional.of("dns-zone"))));
+            }
+            configServer().putLoadBalancers(zone, List.of(new LoadBalancer(deployment.toString(),
+                                                                           deployment.applicationId(),
+                                                                           ClusterSpec.Id.from("default"),
+                                                                           HostName.from("lb-host"),
+                                                                           LoadBalancer.State.active,
+                                                                           Optional.of("dns-zone"))));
+        }
+
         // First step is always a deployment.
         runner.advance(currentRun(job));
 
@@ -493,6 +524,7 @@ public class DeploymentContext {
 
     /** Sets a single endpoint in the routing layer; this matches that required for the tester */
     private DeploymentContext setEndpoints(ZoneId zone, boolean tester) {
+        if (isDirectlyRouted(zone)) return this;
         var id = instanceId;
         if (tester) {
             id = testerId.id();
@@ -542,8 +574,12 @@ public class DeploymentContext {
         assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester));
         configServer().convergeServices(TesterId.of(id.application()).id(), zone);
         runner.advance(currentRun(job));
-        assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester));
-        setTesterEndpoints(zone);
+        if (provisionLoadBalancerIn(zone)) { // Endpoints are available immediately after deployment in directly routed zones
+            assertEquals(succeeded, jobs.run(id).get().stepStatuses().get(Step.installTester));
+        } else {
+            assertEquals(unfinished, jobs.run(id).get().stepStatuses().get(Step.installTester));
+            setTesterEndpoints(zone);
+        }
         runner.advance(currentRun(job));
     }
 
@@ -577,6 +613,15 @@ public class DeploymentContext {
         routing.removeEndpoints(new DeploymentId(TesterId.of(job.application()).id(), zone));
     }
 
+    /** Returns whether a load balancer is expected to be provisioned in given zone */
+    private boolean provisionLoadBalancerIn(ZoneId zone) {
+        return !deferLoadBalancerProvisioning.contains(zone.environment()) && isDirectlyRouted(zone);
+    }
+
+    private boolean isDirectlyRouted(ZoneId zone) {
+        return tester.controller().zoneRegistry().zones().directlyRouted().ids().contains(zone);
+    }
+
     private JobId jobId(JobType type) {
         return new JobId(instanceId, type);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
index 231f8be2ed3..808d9ca05c4 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/endpointcertificates/EndpointCertificateManagerTest.java
@@ -13,15 +13,20 @@ import org.junit.Test;
 
 import java.time.Clock;
 import java.util.Optional;
+import java.util.Set;
 
 import static org.junit.Assert.assertTrue;
 
+/**
+ * @author andreer
+ */
 public class EndpointCertificateManagerTest {
 
     @Test
     public void getEndpointCertificate() {
         SecretStoreMock secretStore = new SecretStoreMock();
         ZoneRegistryMock zoneRegistryMock = new ZoneRegistryMock(SystemName.main);
+        zoneRegistryMock.setDirectlyRouted(Set.copyOf(zoneRegistryMock.zones().all().zones()));
         MockCuratorDb mockCuratorDb = new MockCuratorDb();
         ApplicationCertificateMock applicationCertificateMock = new ApplicationCertificateMock();
         Clock clock = Clock.systemUTC();
@@ -31,4 +36,5 @@ public class EndpointCertificateManagerTest {
         Optional<EndpointCertificateMetadata> endpointCertificateMetadata = endpointCertificateManager.getEndpointCertificateMetadata(instance, id);
         assertTrue(endpointCertificateMetadata.isPresent());
     }
-}
\ No newline at end of file
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
index 66c7334dc2d..eb8af62a0e6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ConfigServerMock.java
@@ -44,6 +44,7 @@ import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -69,7 +70,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
     private final Set<ApplicationId> disallowConvergenceCheckApplications = new HashSet<>();
     private final Version initialVersion = new Version(6, 1, 0);
     private final Set<DeploymentId> suspendedApplications = new HashSet<>();
-    private final Map<ZoneId, List<LoadBalancer>> loadBalancers = new HashMap<>();
+    private final Map<ZoneId, Set<LoadBalancer>> loadBalancers = new HashMap<>();
     private final Map<DeploymentId, List<Log>> warnings = new HashMap<>();
     private final Map<DeploymentId, Set<String>> rotationNames = new HashMap<>();
     private final Map<DeploymentId, List<ClusterMetrics>> clusterMetrics = new HashMap<>();
@@ -260,8 +261,8 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         disallowConvergenceCheckApplications.add(applicationId);
     }
 
-    private List<LoadBalancer> getLoadBalancers(ZoneId zone) {
-        return loadBalancers.getOrDefault(zone, Collections.emptyList());
+    private Set<LoadBalancer> getLoadBalancers(ZoneId zone) {
+        return loadBalancers.getOrDefault(zone, new LinkedHashSet<>());
     }
 
     @Override
@@ -281,8 +282,9 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
         return TesterCloud.Status.SUCCESS;
     }
 
-    public void addLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) {
-        this.loadBalancers.putIfAbsent(zone, new ArrayList<>());
+    /** Add any of given loadBalancers that do not already exist to the load balancers in zone */
+    public void putLoadBalancers(ZoneId zone, List<LoadBalancer> loadBalancers) {
+        this.loadBalancers.putIfAbsent(zone, new LinkedHashSet<>());
         this.loadBalancers.get(zone).addAll(loadBalancers);
     }
 
@@ -366,6 +368,7 @@ public class ConfigServerMock extends AbstractComponent implements ConfigServer
             throw new NotFoundException("No application with id " + applicationId + " exists, cannot deactivate");
         applications.remove(deployment);
         serviceStatus.remove(deployment);
+        removeLoadBalancers(deployment.applicationId(), deployment.zoneId());
     }
 
     // Returns a canned example response
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java
index 269bdcc5dca..611ea4327ed 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneApiMock.java
@@ -8,6 +8,8 @@ import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 
+import java.util.Objects;
+
 /**
  * @author hakonhall
  */
@@ -34,6 +36,10 @@ public class ZoneApiMock implements ZoneApi {
         return newBuilder().with(ZoneId.from(environment, region)).build();
     }
 
+    public static ZoneApiMock from(ZoneId zone) {
+        return newBuilder().with(zone).build();
+    }
+
     @Override
     public SystemName getSystemName() { return systemName; }
 
@@ -46,6 +52,19 @@ public class ZoneApiMock implements ZoneApi {
     @Override
     public String getCloudNativeRegionName() { return cloudNativeRegionName; }
 
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        ZoneApiMock that = (ZoneApiMock) o;
+        return id.equals(that.id);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
     public static class Builder {
         private SystemName systemName = SystemName.defaultSystem();
         private ZoneId id = ZoneId.defaultId();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index 6f8ff39456f..3cb7a13dd54 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -60,7 +60,6 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
                 ZoneApiMock.fromId("prod.us-west-1"),
                 ZoneApiMock.fromId("prod.us-central-1"),
                 ZoneApiMock.fromId("prod.eu-west-1")));
-        setDirectlyRouted(Set.copyOf(this.zones));
     }
 
     public ZoneRegistryMock setDeploymentTimeToLive(ZoneId zone, Duration duration) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
index b7edcef97e4..e5670139b0f 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiTest.java
@@ -6,6 +6,7 @@ import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
 import org.junit.Before;
@@ -13,7 +14,6 @@ import org.junit.Test;
 
 import java.io.File;
 import java.util.List;
-import java.util.Set;
 
 import static org.junit.Assert.assertNotEquals;
 
@@ -36,13 +36,17 @@ public class RoutingApiTest extends ControllerContainerTest {
     @Test
     public void policy_based_routing() {
         var context = deploymentTester.newDeploymentContext();
-
-        // Deploy application
+        // Zones support direct routing
         var westZone = ZoneId.from("prod", "us-west-1");
         var eastZone = ZoneId.from("prod", "us-east-3");
+        deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone),
+                                                                             ZoneApiMock.from(eastZone));
+
+        // Deploy application
         var applicationPackage = new ApplicationPackageBuilder()
                 .region(westZone.region())
                 .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
                 .build();
         context.submit(applicationPackage).deploy();
         context.addRoutingPolicy(westZone, true);
@@ -56,7 +60,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets deployment out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/deployment-status-out.json"));
@@ -64,7 +68,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets deployment in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/deployment-status-in.json"));
@@ -77,7 +81,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets zone out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/zone-status-out.json"));
@@ -85,7 +89,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets zone in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("policy/zone-status-in.json"));
@@ -93,8 +97,6 @@ public class RoutingApiTest extends ControllerContainerTest {
 
     @Test
     public void rotation_based_routing() {
-        // No zones support direct routing
-        deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(Set.of());
         // Deploy application
         var context = deploymentTester.newDeploymentContext();
         var westZone = ZoneId.from("prod", "us-west-1");
@@ -102,7 +104,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         var applicationPackage = new ApplicationPackageBuilder()
                 .region(westZone.region())
                 .region(eastZone.region())
-                .endpoint("default", "qrs", eastZone.region().value(), westZone.region().value())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
                 .build();
         context.submit(applicationPackage).deploy();
 
@@ -116,7 +118,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets deployment out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/deployment-status-out.json"));
@@ -124,7 +126,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets deployment in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/deployment-status-in.json"));
@@ -137,7 +139,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets zone out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/zone-status-out.json"));
@@ -145,15 +147,31 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets zone in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for deployments in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("rotation/zone-status-in.json"));
+    }
+
+    // TODO(mpolden): Remove this once a zone supports either of routing policy and rotation
+    @Test
+    public void mixed_routing() {
+        // Deploy application
+        var context = deploymentTester.newDeploymentContext();
+        var westZone = ZoneId.from("prod", "us-west-1");
+        var eastZone = ZoneId.from("prod", "us-east-3");
+        var applicationPackage = new ApplicationPackageBuilder()
+                .region(westZone.region())
+                .region(eastZone.region())
+                .endpoint("default", "default", eastZone.region().value(), westZone.region().value())
+                .build();
+        context.submit(applicationPackage).deploy();
 
-        // TODO(mpolden): Remove the following once a zone supports either of routing policy and rotation
+        // Assign policy in one zone
+        deploymentTester.controllerTester().zoneRegistry().setDirectlyRouted(ZoneApiMock.from(westZone));
+        context.addRoutingPolicy(westZone, true);
 
         // GET status with both policy and rotation assigned
-        context.addRoutingPolicy(westZone, true);
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("multi-status-initial.json"));
@@ -161,7 +179,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // POST sets deployment out
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.POST),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'out'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to OUT\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("multi-status-out.json"));
@@ -169,7 +187,7 @@ public class RoutingApiTest extends ControllerContainerTest {
         // DELETE sets deployment in
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/inactive/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.DELETE),
-                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to 'in'\"}");
+                              "{\"message\":\"Set global routing status for tenant.application in prod.us-west-1 to IN\"}");
         tester.assertResponse(operatorRequest("http://localhost:8080/routing/v1/status/tenant/tenant/application/application/instance/default/environment/prod/region/us-west-1",
                                               "", Request.Method.GET),
                               new File("multi-status-in.json"));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
index a15a0cc8a99..cdf3e093f97 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/multi-status-initial.json
@@ -6,7 +6,7 @@
       "environment": "prod",
       "region": "us-west-1",
       "status": "in",
-      "agent": "operator",
+      "agent": "unknown",
       "changedAt": "(ignore)"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
index 22ed079b501..1eb4523c344 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/systemflags/SystemFlagsDeployerTest.java
@@ -111,8 +111,6 @@ public class SystemFlagsDeployerTest {
 
         SystemFlagsDeployResult result = deployer.deployFlags(archive, false);
 
-        System.out.println(result);
-
         assertThat(result.errors()).containsOnly(
                 OperationError.listFailed(exception.getMessage(), prodUsWest1Target));
         assertThat(result.flagChanges()).containsOnly(
@@ -123,4 +121,4 @@ public class SystemFlagsDeployerTest {
         return FlagData.deserializeUtf8Json(Files.readAllBytes(Paths.get("src/test/resources/system-flags/" + filename)));
     }
 
-}
\ No newline at end of file
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index ec169633433..c6f9bb81b63 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -1,10 +1,12 @@
 // Copyright 2020 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.routing;
 
+import com.google.common.collect.Sets;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
@@ -30,7 +32,6 @@ import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -72,7 +73,7 @@ public class RoutingPoliciesTest {
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
 
         // Creates alias records
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2);
@@ -91,7 +92,7 @@ public class RoutingPoliciesTest {
                 .build();
         numberOfDeployments++;
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3);
-        context1.submit(applicationPackage2).deploy();
+        context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         // Endpoints are updated to contain cluster in new deployment
         tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3);
@@ -106,7 +107,7 @@ public class RoutingPoliciesTest {
                 .region(zone2.region())
                 .endpoint("r0", "c0")
                 .build();
-        context2.submit(applicationPackage3).deploy();
+        context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
 
         // All endpoints for app1 are removed
@@ -116,7 +117,7 @@ public class RoutingPoliciesTest {
                 .region(zone3.region())
                 .allow(ValidationId.globalEndpointChange)
                 .build();
-        context1.submit(applicationPackage4).deploy();
+        context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0);
         tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0);
@@ -136,7 +137,7 @@ public class RoutingPoliciesTest {
         // Deploy application
         int clustersPerZone = 2;
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         // Deployment creates records and policies for all clusters in all zones
         Set<String> expectedRecords = Set.of(
@@ -149,13 +150,13 @@ public class RoutingPoliciesTest {
         assertEquals(4, tester.policiesOf(context1.instanceId()).size());
 
         // Next deploy does nothing
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         assertEquals(expectedRecords, tester.recordNames());
         assertEquals(4, tester.policiesOf(context1.instanceId()).size());
 
         // Add 1 cluster in each zone and deploy
         tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), zone1, zone2);
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
@@ -169,7 +170,7 @@ public class RoutingPoliciesTest {
 
         // Deploy another application
         tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), zone1, zone2);
-        context2.submit(applicationPackage).deploy();
+        context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
@@ -182,12 +183,12 @@ public class RoutingPoliciesTest {
                 "c0.app2.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app2.tenant1.us-west-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList()));
         assertEquals(4, tester.policiesOf(context2.instanceId()).size());
 
         // Deploy removes cluster from app1
         tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
-        context1.submit(applicationPackage).deploy();
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
                 "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
@@ -228,7 +229,7 @@ public class RoutingPoliciesTest {
                 .region(zone1.region().value())
                 .endpoint("r0", "c0")
                 .build();
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud";
         assertEquals(endpoint + " points to c0 in all regions",
@@ -243,9 +244,9 @@ public class RoutingPoliciesTest {
     public void cluster_endpoints_resolve_from_policies() {
         var tester = new RoutingPoliciesTester();
         var context = tester.newDeploymentContext("tenant1", "app1", "default");
-        tester.provisionLoadBalancers(3, context.instanceId(), zone1);
-        context.submit(applicationPackage).deploy();
-        tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), Collections.emptyList());
+        tester.provisionLoadBalancers(3, context.instanceId(), zone1, zone2);
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        tester.controllerTester().serviceRegistry().routingGeneratorMock().putEndpoints(context.deploymentIdIn(zone1), List.of());
         assertEquals(Map.of(ClusterSpec.Id.from("c0"),
                             URI.create("https://c0.app1.tenant1.us-west-1.vespa.oath.cloud/"),
                             ClusterSpec.Id.from("c1"),
@@ -289,7 +290,8 @@ public class RoutingPoliciesTest {
         tester.controllerTester().serviceRegistry().zoneRegistry()
               .setZones(zoneApi)
               .setDirectlyRouted(zoneApi);
-
+        var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud");
+        assertEquals(prodRecords, tester.recordNames());
 
         // Deploy to dev under different instance
         var devInstance = context.application().id().instance("user");
@@ -300,7 +302,7 @@ public class RoutingPoliciesTest {
 
         // Routing policy is created and DNS is updated
         assertEquals(1, tester.policiesOf(devInstance).size());
-        assertEquals(Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
+        assertEquals(Sets.union(prodRecords, Set.of("c0.user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames());
     }
 
     @Test
@@ -315,7 +317,7 @@ public class RoutingPoliciesTest {
                 .build();
 
         // Application is deployed
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         var expectedRecords = Set.of(
                 "c0.app1.tenant1.us-west-1.vespa.oath.cloud"
         );
@@ -334,10 +336,10 @@ public class RoutingPoliciesTest {
                                             newHostname,
                                             LoadBalancer.State.active,
                                             Optional.of("dns-zone-1"));
-        tester.controllerTester().configServer().addLoadBalancers(zone1, List.of(loadBalancer));
+        tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer));
 
         // Application redeployment preserves DNS record
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         assertEquals(expectedRecords, tester.recordNames());
         assertEquals(1, tester.policiesOf(context.instanceId()).size());
         assertEquals("CNAME points to current load blancer", newHostname.value() + ".",
@@ -357,7 +359,7 @@ public class RoutingPoliciesTest {
                 .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
         // Global DNS record is created
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
@@ -386,7 +388,7 @@ public class RoutingPoliciesTest {
         assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt());
 
         // Next deployment does not affect status
-        context.submit(applicationPackage).deploy();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
@@ -411,7 +413,7 @@ public class RoutingPoliciesTest {
                 .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
-        context.submit(applicationPackage2).deploy();
+        context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1);
 
@@ -422,7 +424,7 @@ public class RoutingPoliciesTest {
                 .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
-        context.submit(applicationPackage3).deploy();
+        context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
     }
@@ -442,7 +444,7 @@ public class RoutingPoliciesTest {
                 .build();
         for (var context : contexts) {
             tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
-            context.submit(applicationPackage).deploy();
+            context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
             tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
         }
 
@@ -520,12 +522,14 @@ public class RoutingPoliciesTest {
 
         public RoutingPoliciesTester(DeploymentTester tester) {
             this.tester = tester;
+            // Make all zones directly routed
+            tester.controllerTester().zoneRegistry().setDirectlyRouted(Set.copyOf(tester.controllerTester().zoneRegistry().zones().all().zones()));
         }
 
         private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) {
             for (ZoneId zone : zones) {
                 tester.configServer().removeLoadBalancers(application, zone);
-                tester.configServer().addLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone));
+                tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, clustersPerZone));
             }
         }