Merge pull request #24512 from vespa-engine/jonmv/cross-instance-region-application-endpoints

Jonmv/cross instance region application endpoints

15 files changed, 305 insertions, 178 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 91bca1e481c..38c06e4dac2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -55,6 +55,9 @@ import java.util.Set;
 import java.util.TreeMap;
 import java.util.stream.Collectors;
 
+import static java.util.stream.Collectors.groupingBy;
+import static java.util.stream.Collectors.toMap;
+
 /**
  * The routing controller encapsulates state and methods for inspecting and manipulating deployment endpoints in a
  * hosted Vespa system.
@@ -155,21 +158,23 @@ public class RoutingController {
         }
         // Add application endpoints
         for (var declaredEndpoint : deploymentSpec.endpoints()) {
-            Map<DeploymentId, Integer> deployments = declaredEndpoint.targets().stream()
-                                                                     .collect(Collectors.toMap(t -> new DeploymentId(application.id().instance(t.instance()),
-                                                                                                                     ZoneId.from(Environment.prod, t.region())),
-                                                                                               t -> t.weight()));
-            // An application endpoint can only target a single zone, so we just pick the zone of any deployment target
-            ZoneId zone = deployments.keySet().iterator().next().zoneId();
-            // Application endpoints are only supported when using direct routing methods
-            RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive;
-            endpoints.add(Endpoint.of(application.id())
-                                  .targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
-                                                     ClusterSpec.Id.from(declaredEndpoint.containerId()),
-                                                     deployments)
-                                  .routingMethod(routingMethod)
-                                  .on(Port.fromRoutingMethod(routingMethod))
-                                  .in(controller.system()));
+            Map<ZoneId, Map<DeploymentId, Integer>> deployments = declaredEndpoint.targets().stream()
+                                                                                  .collect(groupingBy(t -> ZoneId.from(Environment.prod, t.region()),
+                                                                                                      toMap(t -> new DeploymentId(application.id().instance(t.instance()),
+                                                                                                                                  ZoneId.from(Environment.prod, t.region())),
+                                                                                                            t -> t.weight())));
+
+            deployments.forEach((zone, weightedInstances) -> {
+                // Application endpoints are only supported when using direct routing methods
+                RoutingMethod routingMethod = usesSharedRouting(zone) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive;
+                endpoints.add(Endpoint.of(application.id())
+                                      .targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
+                                                         ClusterSpec.Id.from(declaredEndpoint.containerId()),
+                                                         weightedInstances)
+                                      .routingMethod(routingMethod)
+                                      .on(Port.fromRoutingMethod(routingMethod))
+                                      .in(controller.system()));
+            });
         }
         return EndpointList.copyOf(endpoints);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
index f72b6f2e9f0..4d29231f6d4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
@@ -5,11 +5,13 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.Endpoint;
+import com.yahoo.config.application.api.Endpoint.Level;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
+import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
@@ -19,6 +21,7 @@ import com.yahoo.vespa.hosted.controller.versions.VespaVersion;
 
 import java.time.Instant;
 import java.util.ArrayList;
+import java.util.EnumSet;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -90,22 +93,40 @@ public class ApplicationPackageValidator {
         }
     }
 
-    /** Verify that no single endpoint contains regions in different clouds */
+    /** Verify that:
+     * <ul>
+     *     <li>no single endpoint contains regions in different clouds</li>
+     *     <li>application endpoints with different regions must be contained in CGP and AWS</li>
+     * </ul>
+     */
     private void validateEndpointRegions(DeploymentSpec deploymentSpec) {
         for (var instance : deploymentSpec.instances()) {
-            for (var endpoint : instance.endpoints()) {
-                var clouds = new HashSet<CloudName>();
-                for (var region : endpoint.regions()) {
-                    for (ZoneApi zone : controller.zoneRegistry().zones().all().in(Environment.prod).in(region).zones()) {
-                        clouds.add(zone.getCloudName());
-                    }
-                }
-                if (clouds.size() != 1 && !clouds.equals(Set.of(CloudName.GCP, CloudName.AWS))) {
-                    throw new IllegalArgumentException("Endpoint '" + endpoint.endpointId() + "' in " + instance +
-                                                       " cannot contain regions in different clouds: " +
+            validateEndpointRegions(instance.endpoints(), instance);
+        }
+        validateEndpointRegions(deploymentSpec.endpoints(), null);
+    }
+
+    private void validateEndpointRegions(List<Endpoint> endpoints, DeploymentInstanceSpec instance) {
+        for (var endpoint : endpoints) {
+            RegionName[] regions = new HashSet<>(endpoint.regions()).toArray(RegionName[]::new);
+            Set<CloudName> clouds = controller.zoneRegistry().zones().all().in(Environment.prod)
+                                              .in(regions)
+                                              .zones().stream()
+                                              .map(ZoneApi::getCloudName)
+                                              .collect(Collectors.toSet());
+            String endpointString = instance == null ? "Application endpoint '" + endpoint.endpointId() + "'"
+                                                     : "Endpoint '" + endpoint.endpointId() + "' in " + instance;
+            if (Set.of(CloudName.GCP, CloudName.AWS).containsAll(clouds)) { } // Everything is fine!
+            else if (Set.of(CloudName.DEFAULT).containsAll(clouds)) {
+                if (endpoint.level() == Level.application && regions.length != 1) {
+                    throw new IllegalArgumentException(endpointString + " cannot contain different regions: " +
                                                        endpoint.regions().stream().sorted().toList());
                 }
             }
+            else {
+                throw new IllegalArgumentException(endpointString + " cannot contain regions in different clouds: " +
+                                                   endpoint.regions().stream().sorted().toList());
+            }
         }
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index fc0badae9ea..27247c065ed 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -256,24 +256,22 @@ public class RoutingPolicies {
             EndpointList endpoints = controller.routing().declaredEndpointsOf(application)
                                                .scope(Endpoint.Scope.application)
                                                .named(routingId.endpointId());
-            if (endpoints.isEmpty()) continue;
-            if (endpoints.size() > 1) {
-                throw new IllegalArgumentException("Expected at most 1 endpoint with ID '" + routingId.endpointId() +
-                                                   ", got " + endpoints.size());
-            }
-            Endpoint endpoint = endpoints.asList().get(0);
-            for (var policy : routeEntry.getValue()) {
-                for (var target : endpoint.targets()) {
-                    if (!policy.appliesTo(target.deployment())) continue;
-                    if (policy.dnsZone().isEmpty() && policy.canonicalName().isPresent()) continue; // Does not support ALIAS records
-                    ZoneRoutingPolicy zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
-
-                    Set<Target> activeTargets = targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>());
-                    Set<Target> inactiveTargets = inactiveTargetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>());
-                    if (isConfiguredOut(zonePolicy, policy, inactiveZones)) {
-                        inactiveTargets.add(Target.weighted(policy, target));
-                    } else {
-                        activeTargets.add(Target.weighted(policy, target));
+            for (Endpoint endpoint : endpoints) {
+                for (var policy : routeEntry.getValue()) {
+                    for (var target : endpoint.targets()) {
+                        if (!policy.appliesTo(target.deployment())) continue;
+                        if (policy.dnsZone().isEmpty() && policy.canonicalName().isPresent())
+                            continue; // Does not support ALIAS records
+                        ZoneRoutingPolicy zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
+
+                        Set<Target> activeTargets = targetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>());
+                        Set<Target> inactiveTargets = inactiveTargetsByEndpoint.computeIfAbsent(endpoint, (k) -> new LinkedHashSet<>());
+                        if (isConfiguredOut(zonePolicy, policy, inactiveZones)) {
+                            inactiveTargets.add(Target.weighted(policy, target));
+                        }
+                        else {
+                            activeTargets.add(Target.weighted(policy, target));
+                        }
                     }
                 }
             }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
index cd3d6ca7531..43a4ac0f6fc 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/ControllerTest.java
@@ -62,6 +62,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.OptionalInt;
 import java.util.Set;
+import java.util.TreeSet;
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -72,6 +73,7 @@ import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.pro
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.productionUsWest1;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.stagingTest;
 import static com.yahoo.vespa.hosted.controller.deployment.DeploymentContext.systemTest;
+import static java.util.Comparator.comparing;
 import static org.junit.jupiter.api.Assertions.assertArrayEquals;
 import static org.junit.jupiter.api.Assertions.assertEquals;
 import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -344,7 +346,7 @@ public class ControllerTest {
 
         List<String> globalDnsNames = tester.controller().routing().readDeclaredEndpointsOf(context.instanceId())
                 .scope(Endpoint.Scope.global)
-                .sortedBy(Comparator.comparing(Endpoint::dnsName))
+                .sortedBy(comparing(Endpoint::dnsName))
                 .mapToList(Endpoint::dnsName);
         assertEquals(List.of("app1.tenant1.global.vespa.oath.cloud"),
                 globalDnsNames);
@@ -636,63 +638,107 @@ public class ControllerTest {
         var context = tester.newDeploymentContext(beta);
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .instances("beta,main")
-                .region("us-west-1")
                 .region("us-east-3")
-                .applicationEndpoint("a", "default", "us-west-1",
-                        Map.of(beta.instance(), 2,
-                                main.instance(), 8))
-                .applicationEndpoint("b", "default", "us-west-1",
-                        Map.of(beta.instance(), 1,
-                                main.instance(), 1))
-                .applicationEndpoint("c", "default", "us-east-3",
-                        Map.of(beta.instance(), 4,
-                                main.instance(), 6))
+                .region("us-west-1")
+                .region("aws-us-east-1a")
+                .region("aws-us-east-1b")
+                .applicationEndpoint("a", "default",
+                                     Map.of("aws-us-east-1a", Map.of(beta.instance(), 2,
+                                                                     main.instance(), 8),
+                                            "aws-us-east-1b", Map.of(main.instance(), 1)))
+                .applicationEndpoint("b", "default", "aws-us-east-1a",
+                                     Map.of(beta.instance(), 1,
+                                            main.instance(), 1))
+                .applicationEndpoint("c", "default", "aws-us-east-1b",
+                                     Map.of(beta.instance(), 4))
+                .applicationEndpoint("d", "default", "us-west-1",
+                                     Map.of(main.instance(), 7,
+                                            beta.instance(), 3))
+                .applicationEndpoint("e", "default", "us-east-3",
+                                     Map.of(main.instance(), 3))
                 .build();
         context.submit(applicationPackage).deploy();
 
-        ZoneId usWest = ZoneId.from("prod", "us-west-1");
-        ZoneId usEast = ZoneId.from("prod", "us-east-3");
+        ZoneId east3 = ZoneId.from("prod", "us-east-3");
+        ZoneId west1 = ZoneId.from("prod", "us-west-1");
+        ZoneId east1a = ZoneId.from("prod", "aws-us-east-1a");
+        ZoneId east1b = ZoneId.from("prod", "aws-us-east-1b");
         // Expected container endpoints are passed to each deployment
         Map<DeploymentId, Map<String, Integer>> deploymentEndpoints = Map.of(
-                new DeploymentId(beta, usWest), Map.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", 2,
-                        "b.app1.tenant1.us-west-1-r.vespa.oath.cloud", 1),
-                new DeploymentId(main, usWest), Map.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud", 8,
-                        "b.app1.tenant1.us-west-1-r.vespa.oath.cloud", 1),
-                new DeploymentId(beta, usEast), Map.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud", 4),
-                new DeploymentId(main, usEast), Map.of("c.app1.tenant1.us-east-3-r.vespa.oath.cloud", 6)
+                new DeploymentId(beta, east3), Map.of(),
+                new DeploymentId(main, east3), Map.of("e.app1.tenant1.us-east-3-r.vespa.oath.cloud", 3),
+                new DeploymentId(beta, west1), Map.of("d.app1.tenant1.us-west-1-r.vespa.oath.cloud", 3),
+                new DeploymentId(main, west1), Map.of("d.app1.tenant1.us-west-1-r.vespa.oath.cloud", 7),
+                new DeploymentId(beta, east1a), Map.of("a.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud", 2,
+                                                       "b.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud", 1),
+                new DeploymentId(main, east1a), Map.of("a.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud", 8,
+                                                       "b.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud", 1),
+                new DeploymentId(beta, east1b), Map.of("c.app1.tenant1.aws-us-east-1b-r.vespa.oath.cloud", 4),
+                new DeploymentId(main, east1b), Map.of("a.app1.tenant1.aws-us-east-1b-r.vespa.oath.cloud", 1)
         );
         deploymentEndpoints.forEach((deployment, endpoints) -> {
             Set<ContainerEndpoint> expected = endpoints.entrySet().stream()
-                    .map(kv -> new ContainerEndpoint("default", "application",
-                            List.of(kv.getKey()),
-                            OptionalInt.of(kv.getValue()),
-                            RoutingMethod.sharedLayer4))
-                    .collect(Collectors.toSet());
+                                                       .map(kv -> new ContainerEndpoint("default", "application",
+                                                                                        List.of(kv.getKey()),
+                                                                                        OptionalInt.of(kv.getValue()),
+                                                                                        tester.controller().zoneRegistry().routingMethod(deployment.zoneId())))
+                                                       .collect(Collectors.toSet());
             assertEquals(expected,
-                    tester.configServer().containerEndpoints().get(deployment),
-                    "Endpoint names for " + deployment + " are passed to config server");
+                         tester.configServer().containerEndpoints().get(deployment),
+                         "Endpoint names for " + deployment + " are passed to config server");
         });
         context.flushDnsUpdates();
 
         // DNS records are created for each endpoint
         Set<Record> records = tester.controllerTester().nameService().records();
-        assertEquals(Set.of(new Record(Record.Type.CNAME,
-                                RecordName.from("a.app1.tenant1.us-west-1-r.vespa.oath.cloud"),
-                                RecordData.from("vip.prod.us-west-1.")),
-                        new Record(Record.Type.CNAME,
-                                RecordName.from("b.app1.tenant1.us-west-1-r.vespa.oath.cloud"),
-                                RecordData.from("vip.prod.us-west-1.")),
-                        new Record(Record.Type.CNAME,
-                                RecordName.from("c.app1.tenant1.us-east-3-r.vespa.oath.cloud"),
-                                RecordData.from("vip.prod.us-east-3."))),
-                records);
+        assertEquals(new TreeSet<>(Set.of(new Record(Record.Type.CNAME,
+                                                     RecordName.from("beta.app1.tenant1.aws-us-east-1a.vespa.oath.cloud"),
+                                                     RecordData.from("lb-0--tenant1.app1.beta--prod.aws-us-east-1a.")),
+                                          new Record(Record.Type.CNAME,
+                                                     RecordName.from("beta.app1.tenant1.aws-us-east-1b.vespa.oath.cloud"),
+                                                     RecordData.from("lb-0--tenant1.app1.beta--prod.aws-us-east-1b.")),
+                                          new Record(Record.Type.CNAME,
+                                                     RecordName.from("main.app1.tenant1.aws-us-east-1a.vespa.oath.cloud"),
+                                                     RecordData.from("lb-0--tenant1.app1.main--prod.aws-us-east-1a.")),
+                                          new Record(Record.Type.CNAME,
+                                                     RecordName.from("main.app1.tenant1.aws-us-east-1b.vespa.oath.cloud"),
+                                                     RecordData.from("lb-0--tenant1.app1.main--prod.aws-us-east-1b.")),
+                                          new Record(Record.Type.ALIAS,
+                                                     RecordName.from("a.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud"),
+                                                     RecordData.from("weighted/lb-0--tenant1.app1.beta--prod.aws-us-east-1a/dns-zone-1/prod.aws-us-east-1a/2")),
+                                          new Record(Record.Type.ALIAS,
+                                                     RecordName.from("a.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud"),
+                                                     RecordData.from("weighted/lb-0--tenant1.app1.main--prod.aws-us-east-1a/dns-zone-1/prod.aws-us-east-1a/8")),
+                                          new Record(Record.Type.ALIAS,
+                                                     RecordName.from("a.app1.tenant1.aws-us-east-1b-r.vespa.oath.cloud"),
+                                                     RecordData.from("weighted/lb-0--tenant1.app1.main--prod.aws-us-east-1b/dns-zone-1/prod.aws-us-east-1b/1")),
+                                          new Record(Record.Type.ALIAS,
+                                                     RecordName.from("b.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud"),
+                                                     RecordData.from("weighted/lb-0--tenant1.app1.beta--prod.aws-us-east-1a/dns-zone-1/prod.aws-us-east-1a/1")),
+                                          new Record(Record.Type.ALIAS,
+                                                     RecordName.from("b.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud"),
+                                                     RecordData.from("weighted/lb-0--tenant1.app1.main--prod.aws-us-east-1a/dns-zone-1/prod.aws-us-east-1a/1")),
+                                          new Record(Record.Type.ALIAS,
+                                                     RecordName.from("c.app1.tenant1.aws-us-east-1b-r.vespa.oath.cloud"),
+                                                     RecordData.from("weighted/lb-0--tenant1.app1.beta--prod.aws-us-east-1b/dns-zone-1/prod.aws-us-east-1b/4")),
+                                          new Record(Record.Type.CNAME,
+                                                     RecordName.from("d.app1.tenant1.us-west-1-r.vespa.oath.cloud"),
+                                                     RecordData.from("vip.prod.us-west-1.")),
+                                          new Record(Record.Type.CNAME,
+                                                     RecordName.from("e.app1.tenant1.us-east-3-r.vespa.oath.cloud"),
+                                                     RecordData.from("vip.prod.us-east-3.")))),
+                     new TreeSet<>(records));
         List<String> endpointDnsNames = tester.controller().routing().declaredEndpointsOf(context.application())
-                .scope(Endpoint.Scope.application)
-                .mapToList(Endpoint::dnsName);
-        assertEquals(List.of("a.app1.tenant1.us-west-1-r.vespa.oath.cloud",
-                        "b.app1.tenant1.us-west-1-r.vespa.oath.cloud",
-                        "c.app1.tenant1.us-east-3-r.vespa.oath.cloud"),
-                endpointDnsNames);
+                                              .scope(Endpoint.Scope.application)
+                                              .sortedBy(comparing(Endpoint::dnsName))
+                                              .mapToList(Endpoint::dnsName);
+        assertEquals(List.of("a.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud",
+                             "a.app1.tenant1.aws-us-east-1b-r.vespa.oath.cloud",
+                             "b.app1.tenant1.aws-us-east-1a-r.vespa.oath.cloud",
+                             "c.app1.tenant1.aws-us-east-1b-r.vespa.oath.cloud",
+                             "d.app1.tenant1.us-west-1-r.vespa.oath.cloud",
+                             "e.app1.tenant1.us-east-3-r.vespa.oath.cloud"),
+                     endpointDnsNames);
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java
index 274cf3c5867..b06b4eb0cfa 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/certificate/EndpointCertificatesTest.java
@@ -191,7 +191,7 @@ public class EndpointCertificatesTest {
 
     @Test
     void reprovisions_certificate_with_added_sans_when_deploying_to_new_zone() {
-        ZoneId testZone = tester.zoneRegistry().zones().all().routingMethod(RoutingMethod.exclusive).in(Environment.prod).zones().stream().skip(1).findFirst().orElseThrow().getId();
+        ZoneId testZone = ZoneId.from("prod.ap-northeast-1");
 
         mockCuratorDb.writeEndpointCertificateMetadata(testInstance.id(), new EndpointCertificateMetadata(testKeyName, testCertName, -1, 0, "original-request-uuid", Optional.of("leaf-request-uuid"), expectedSans, "mockCa", Optional.empty(), Optional.empty()));
         secretStore.setSecret("vespa.tls.default.default.default-key", KeyUtils.toPem(testKeyPair.getPrivate()), -1);
@@ -243,14 +243,13 @@ public class EndpointCertificatesTest {
                 .region(zone1.region())
                 .region(zone2.region())
                 .applicationEndpoint("a", "qrs", zone2.region().value(),
-                        Map.of(InstanceName.from("beta"), 2,
-                                InstanceName.from("main"), 8))
+                                     Map.of(InstanceName.from("beta"), 2))
                 .applicationEndpoint("b", "qrs", zone2.region().value(),
-                        Map.of(InstanceName.from("beta"), 1,
-                                InstanceName.from("main"), 1))
-                .applicationEndpoint("c", "qrs", zone1.region().value(),
-                        Map.of(InstanceName.from("beta"), 4,
-                                InstanceName.from("main"), 6))
+                                     Map.of(InstanceName.from("beta"), 1))
+                .applicationEndpoint("c", "qrs",
+                                     Map.of(zone1.region().value(), Map.of(InstanceName.from("beta"), 4,
+                                                                           InstanceName.from("main"), 6),
+                                            zone2.region().value(), Map.of(InstanceName.from("main"), 2)))
                 .build();
         ControllerTester tester = new ControllerTester(SystemName.Public);
         EndpointCertificateValidatorImpl endpointCertificateValidator = new EndpointCertificateValidatorImpl(secretStore, clock);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index 7909651716a..7fb85af0d4a 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -120,17 +120,24 @@ public class ApplicationPackageBuilder {
 
     public ApplicationPackageBuilder applicationEndpoint(String id, String containerId, String region,
                                                          Map<InstanceName, Integer> instanceWeights) {
+        return applicationEndpoint(id, containerId, Map.of(region, instanceWeights));
+    }
+
+    public ApplicationPackageBuilder applicationEndpoint(String id, String containerId,
+                                                         Map<String, Map<InstanceName, Integer>> instanceWeights) {
         if (instanceWeights.isEmpty()) throw new IllegalArgumentException("At least one instance must be given");
         applicationEndpointsBody.append("    <endpoint");
         applicationEndpointsBody.append(" id='").append(id).append("'");
         applicationEndpointsBody.append(" container-id='").append(containerId).append("'");
-        applicationEndpointsBody.append(" region='").append(region).append("'");
         applicationEndpointsBody.append(">\n");
-        for (var kv : new TreeMap<>(instanceWeights).entrySet()) {
-            applicationEndpointsBody.append("      <instance weight='").append(kv.getValue().toString()).append("'>")
-                                    .append(kv.getKey().value())
-                                    .append("</instance>\n");
-        }
+        new TreeMap<>(instanceWeights).forEach((region, instances) -> {
+            new TreeMap<>(instances).forEach((instance, weight) -> {
+                applicationEndpointsBody.append("      <instance weight='").append(weight.toString()).append("' region='").append(region).append("'>")
+                                        .append(instance)
+                                        .append("</instance>\n");
+
+            });
+        });
         applicationEndpointsBody.append("    </endpoint>\n");
         return this;
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 73630488969..6d1e5cebe0e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -1305,11 +1305,11 @@ public class DeploymentTriggerTest {
                                     </steps>
                                     <steps>
                                         <delay hours='3' />
-                                        <region active='true'>aws-us-east-1a</region>
+                                        <region active='true'>us-central-1</region>
                                         <parallel>
                                             <region active='true' athenz-service='no-service'>ap-northeast-1</region>
                                             <region active='true'>ap-northeast-2</region>
-                                            <test>aws-us-east-1a</test>
+                                            <test>us-central-1</test>
                                         </parallel>
                                     </steps>
                                     <delay hours='3' minutes='30' />
@@ -1388,10 +1388,10 @@ public class DeploymentTriggerTest {
         assertEquals(List.of(), tester.jobs().active());
 
         tester.clock().advance(Duration.ofHours(1));
-        app1.runJob(productionAwsUsEast1a);
+        app1.runJob(productionUsCentral1);
         tester.triggerJobs();
         assertEquals(3, tester.jobs().active().size());
-        app1.runJob(testAwsUsEast1a);
+        app1.runJob(testUsCentral1);
         tester.triggerJobs();
         assertEquals(2, tester.jobs().active().size());
         app1.runJob(productionApNortheast2);
@@ -1448,11 +1448,11 @@ public class DeploymentTriggerTest {
         tester.clock().advance(Duration.ofHours(2));
         app1.runJob(productionEuWest1);
         tester.clock().advance(Duration.ofHours(1));
-        app1.runJob(productionAwsUsEast1a);
-        app1.runJob(testAwsUsEast1a);
+        app1.runJob(productionUsCentral1);
+        app1.runJob(testUsCentral1);
         tester.clock().advance(Duration.ofSeconds(1));
-        app1.runJob(productionAwsUsEast1a); // R
-        app1.runJob(testAwsUsEast1a);       // R
+        app1.runJob(productionUsCentral1); // R
+        app1.runJob(testUsCentral1);       // R
         app1.runJob(productionApNortheast2);
         app1.runJob(productionApNortheast1);
         tester.clock().advance(Duration.ofHours(1));
@@ -1996,8 +1996,8 @@ public class DeploymentTriggerTest {
     @Test
     void mixedDirectAndPipelineJobsInProduction() {
         ApplicationPackage cdPackage = new ApplicationPackageBuilder().region("us-east-3")
-                .region("aws-us-east-1a")
-                .build();
+                                                                      .region("aws-us-east-1a")
+                                                                      .build();
         ControllerTester wrapped = new ControllerTester(cd);
         wrapped.upgradeSystem(Version.fromString("6.1"));
         wrapped.computeVersionStatus();
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
index 0b520328cb2..a3ea56ccc72 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/integration/ZoneRegistryMock.java
@@ -71,7 +71,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
                                  ZoneApiMock.fromId("dev.us-east-1"),
                                  ZoneApiMock.fromId("dev.aws-us-east-2a"),
                                  ZoneApiMock.fromId("perf.us-east-3"),
-                                 ZoneApiMock.fromId("prod.aws-us-east-1a"),
+                                 ZoneApiMock.newBuilder().withId("prod.aws-us-east-1a").withCloud("aws").build(),
+                                 ZoneApiMock.newBuilder().withId("prod.aws-us-east-1b").withCloud("aws").build(),
                                  ZoneApiMock.fromId("prod.ap-northeast-1"),
                                  ZoneApiMock.fromId("prod.ap-northeast-2"),
                                  ZoneApiMock.fromId("prod.ap-southeast-1"),
@@ -79,7 +80,8 @@ public class ZoneRegistryMock extends AbstractComponent implements ZoneRegistry
                                  ZoneApiMock.fromId("prod.us-west-1"),
                                  ZoneApiMock.fromId("prod.us-central-1"),
                                  ZoneApiMock.fromId("prod.eu-west-1"));
-            setRoutingMethod(this.zones, RoutingMethod.sharedLayer4);
+            for (ZoneApi zone : this.zones)
+                setRoutingMethod(zone, zone.getCloudName().equals(CloudName.DEFAULT) ? RoutingMethod.sharedLayer4 : RoutingMethod.exclusive);
         }
     }
 
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
index 2608a722e49..454a1ac9ee9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/OsVersionStatusUpdaterTest.java
@@ -44,9 +44,20 @@ public class OsVersionStatusUpdaterTest {
         statusUpdater.maintain();
 
         var osVersions = tester.controller().osVersionStatus().versions();
-        assertEquals(2, osVersions.size());
+        assertEquals(3, osVersions.size());
         assertFalse(osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty(), "All nodes on unknown version");
         assertTrue(osVersions.get(new OsVersion(version1, cloud)).isEmpty(), "No nodes on current target");
+
+        CloudName otherCloud = CloudName.AWS;
+        tester.controller().upgradeOsIn(otherCloud, version1, Duration.ZERO, false);
+        statusUpdater.maintain();
+
+        osVersions = tester.controller().osVersionStatus().versions();
+        assertEquals(4, osVersions.size()); // 2 in cloud, 2 in otherCloud.
+        assertFalse(osVersions.get(new OsVersion(Version.emptyVersion, cloud)).isEmpty(), "All nodes on unknown version");
+        assertTrue(osVersions.get(new OsVersion(version1, cloud)).isEmpty(), "No nodes on current target");
+        assertFalse(osVersions.get(new OsVersion(Version.emptyVersion, otherCloud)).isEmpty(), "All nodes on unknown version");
+        assertTrue(osVersions.get(new OsVersion(version1, otherCloud)).isEmpty(), "No nodes on current target");
     }
 
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
index 9148ad36d46..80bcbc7ee7e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/ControllerApiTest.java
@@ -51,7 +51,7 @@ public class ControllerApiTest extends ControllerContainerTest {
 
         // GET a list of all maintenance jobs
         tester.assertResponse(authenticatedRequest("http://localhost:8080/controller/v1/maintenance/", "", Request.Method.GET),
-                new File("maintenance.json"));
+                              new File("maintenance.json"));
     }
 
     @Test
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
index 6226e94f7b4..48771ac064b 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/controller/responses/maintenance.json
@@ -16,6 +16,9 @@
       "name": "ArtifactExpirer"
     },
     {
+      "name": "AwsOsUpgrader"
+    },
+    {
       "name": "BillingDatabaseMaintainer"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
index 6ef57aaed21..864fb628e92 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/deployment/responses/root.json
@@ -365,6 +365,8 @@
     "test-ap-southeast-1",
     "production-aws-us-east-1a",
     "test-aws-us-east-1a",
+    "production-aws-us-east-1b",
+    "test-aws-us-east-1b",
     "production-eu-west-1",
     "test-eu-west-1",
     "production-us-central-1",
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json
index 1e06b279873..d8a64afb34e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/discovery/environment.json
@@ -22,6 +22,9 @@
       "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1a/"
     },
     {
+      "url": "http://localhost:8080/routing/v1/status/environment/prod/region/aws-us-east-1b/"
+    },
+    {
       "url": "http://localhost:8080/routing/v1/status/environment/prod/region/eu-west-1/"
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json
index 1711bb1f856..2d4553978c6 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/routing/responses/recursion/environment.json
@@ -41,11 +41,19 @@
       "changedAt": 0
     },
     {
-      "routingMethod": "sharedLayer4",
+      "routingMethod": "exclusive",
       "environment": "prod",
       "region": "aws-us-east-1a",
       "status": "in",
-      "agent": "operator",
+      "agent": "system",
+      "changedAt": 0
+    },
+    {
+      "routingMethod": "exclusive",
+      "environment": "prod",
+      "region": "aws-us-east-1b",
+      "status": "in",
+      "agent": "system",
       "changedAt": 0
     },
     {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 29834863976..7b00e7040b5 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -26,6 +26,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
@@ -62,6 +63,8 @@ public class RoutingPoliciesTest {
     private static final ZoneId zone2 = ZoneId.from("prod", "us-central-1");
     private static final ZoneId zone3 = ZoneId.from("prod", "aws-us-east-1a");
     private static final ZoneId zone4 = ZoneId.from("prod", "aws-us-east-1b");
+    private static final ZoneId zone5 = ZoneId.from("prod", "north");
+    private static final ZoneId zone6 = ZoneId.from("prod", "south");
 
     private static final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
                                                                                             .region(zone2.region())
@@ -399,15 +402,15 @@ public class RoutingPoliciesTest {
         context.submit(applicationPackage).deploy();
 
         tester.assertTargets(context.instanceId(), EndpointId.defaultId(),
-                ClusterSpec.Id.from("default"), 0,
-                Map.of(zone1, 1L, zone2, 1L));
+                             ClusterSpec.Id.from("default"), 0,
+                             Map.of(zone1, 1L, zone2, 1L));
         assertEquals(Set.of("app1.tenant1.aws-eu-west-1.w.vespa-app.cloud",
-                        "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
-                        "app1.tenant1.aws-us-east-1.w.vespa-app.cloud",
-                        "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
-                        "app1.tenant1.g.vespa-app.cloud"),
-                tester.recordNames(),
-                "Registers expected DNS names");
+                            "app1.tenant1.aws-eu-west-1a.z.vespa-app.cloud",
+                            "app1.tenant1.aws-us-east-1.w.vespa-app.cloud",
+                            "app1.tenant1.aws-us-east-1c.z.vespa-app.cloud",
+                            "app1.tenant1.g.vespa-app.cloud"),
+                     tester.recordNames(),
+                     "Registers expected DNS names");
     }
 
     @Test
@@ -419,8 +422,8 @@ public class RoutingPoliciesTest {
         var zone = ZoneId.from("dev", "us-east-1");
         var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
         tester.controllerTester().serviceRegistry().zoneRegistry()
-                .setZones(zoneApi)
-                .exclusiveRoutingIn(zoneApi);
+              .setZones(zoneApi)
+              .exclusiveRoutingIn(zoneApi);
 
         // Deploy to dev
         context.runJob(zone, emptyApplicationPackage);
@@ -735,16 +738,17 @@ public class RoutingPoliciesTest {
         DeploymentContext mainContext = tester.newDeploymentContext(mainInstance);
         var applicationPackage = applicationPackageBuilder()
                 .instances("beta,main")
-                .region(zone1.region())
-                .region(zone2.region())
-                .applicationEndpoint("a0", "c0", "us-west-1",
-                        Map.of(betaInstance.instance(), 2,
-                                mainInstance.instance(), 8))
-                .applicationEndpoint("a1", "c1", "us-central-1",
-                        Map.of(betaInstance.instance(), 4,
-                                mainInstance.instance(), 6))
+                .region(zone5.region())
+                .region(zone6.region())
+                .applicationEndpoint("a0", "c0",
+                                     Map.of(zone5.region().value(), Map.of(betaInstance.instance(), 2,
+                                                                           mainInstance.instance(), 8),
+                                            zone6.region().value(), Map.of(mainInstance.instance(), 7)))
+                .applicationEndpoint("a1", "c1", zone6.region().value(),
+                                     Map.of(betaInstance.instance(), 4,
+                                            mainInstance.instance(), 6))
                 .build();
-        for (var zone : List.of(zone1, zone2)) {
+        for (var zone : List.of(zone5, zone6)) {
             tester.provisionLoadBalancers(2, betaInstance, zone);
             tester.provisionLoadBalancers(2, mainInstance, zone);
         }
@@ -753,50 +757,53 @@ public class RoutingPoliciesTest {
         betaContext.submit(applicationPackage).deploy();
 
         // Application endpoint points to both instances with correct weights
-        DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1);
-        DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1);
-        DeploymentId betaZone2 = betaContext.deploymentIdIn(zone2);
-        DeploymentId mainZone2 = mainContext.deploymentIdIn(zone2);
+        DeploymentId betaZone5 = betaContext.deploymentIdIn(zone5);
+        DeploymentId mainZone5 = mainContext.deploymentIdIn(zone5);
+        DeploymentId betaZone6 = betaContext.deploymentIdIn(zone6);
+        DeploymentId mainZone6 = mainContext.deploymentIdIn(zone6);
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(betaZone1, 2,
-                        mainZone1, 8));
+                             Map.of(betaZone5, 2,
+                                    mainZone5, 8,
+                                    mainZone6, 7));
         tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
-                Map.of(betaZone2, 4,
-                        mainZone2, 6));
+                             Map.of(betaZone6, 4,
+                                    mainZone6, 6));
 
         // Weights are updated
         applicationPackage = applicationPackageBuilder()
                 .instances("beta,main")
-                .region(zone1.region())
-                .region(zone2.region())
-                .applicationEndpoint("a0", "c0", "us-west-1",
-                        Map.of(betaInstance.instance(), 3,
-                                mainInstance.instance(), 7))
-                .applicationEndpoint("a1", "c1", "us-central-1",
-                        Map.of(betaInstance.instance(), 1,
-                                mainInstance.instance(), 9))
+                .region(zone5.region())
+                .region(zone6.region())
+                .applicationEndpoint("a0", "c0",
+                                     Map.of(zone5.region().value(), Map.of(betaInstance.instance(), 3,
+                                                                           mainInstance.instance(), 7),
+                                            zone6.region().value(), Map.of(mainInstance.instance(), 2)))
+                .applicationEndpoint("a1", "c1", zone6.region().value(),
+                                     Map.of(betaInstance.instance(), 1,
+                                            mainInstance.instance(), 9))
                 .build();
         betaContext.submit(applicationPackage).deploy();
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(betaZone1, 3,
-                        mainZone1, 7));
+                             Map.of(betaZone5, 3,
+                                    mainZone5, 7,
+                                    mainZone6, 2));
         tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
-                Map.of(betaZone2, 1,
-                        mainZone2, 9));
+                             Map.of(betaZone6, 1,
+                                    mainZone6, 9));
 
         // An endpoint is removed
         applicationPackage = applicationPackageBuilder()
                 .instances("beta,main")
-                .region(zone1.region())
-                .region(zone2.region())
-                .applicationEndpoint("a0", "c0", "us-west-1",
-                        Map.of(betaInstance.instance(), 1))
+                .region(zone5.region())
+                .region(zone6.region())
+                .applicationEndpoint("a0", "c0", zone5.region().value(),
+                                     Map.of(betaInstance.instance(), 1))
                 .build();
         betaContext.submit(applicationPackage).deploy();
 
         // Application endpoints now point to a single instance
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(betaZone1, 1));
+                             Map.of(betaZone5, 1));
         assertTrue(tester.controllerTester().controller().routing()
                         .readDeclaredEndpointsOf(application)
                         .named(EndpointId.of("a1")).isEmpty(),
@@ -814,50 +821,59 @@ public class RoutingPoliciesTest {
         DeploymentContext mainContext = tester.newDeploymentContext(mainInstance);
         var applicationPackage = applicationPackageBuilder()
                 .instances("beta,main")
-                .region(zone1.region())
-                .applicationEndpoint("a0", "c0", "us-west-1",
-                        Map.of(betaInstance.instance(), 2,
-                                mainInstance.instance(), 8))
+                .region(zone5.region())
+                .region(zone6.region())
+                .applicationEndpoint("a0", "c0", Map.of(zone5.region().value(), Map.of(betaInstance.instance(), 2,
+                                                                                       mainInstance.instance(), 8),
+                                                        zone6.region().value(), Map.of(mainInstance.instance(), 9)))
                 .build();
-        tester.provisionLoadBalancers(1, betaInstance, zone1);
-        tester.provisionLoadBalancers(1, mainInstance, zone1);
+        tester.provisionLoadBalancers(1, betaInstance, zone5);
+        tester.provisionLoadBalancers(1, mainInstance, zone5);
+        tester.provisionLoadBalancers(1, mainInstance, zone6);
 
         // Deploy both instances
         betaContext.submit(applicationPackage).deploy();
 
         // Application endpoint points to both instances with correct weights
-        DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1);
-        DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1);
+        DeploymentId betaZone1 = betaContext.deploymentIdIn(zone5);
+        DeploymentId mainZone1 = mainContext.deploymentIdIn(zone5);
+        DeploymentId mainZone2 = mainContext.deploymentIdIn(zone6);
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(betaZone1, 2,
-                        mainZone1, 8));
+                             Map.of(betaZone1, 2,
+                                    mainZone1, 8,
+                                    mainZone2, 9));
 
         // Changing routing status removes deployment from DNS
         tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
         betaContext.flushDnsUpdates();
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(betaZone1, 2));
+                             Map.of(betaZone1, 2,
+                                    mainZone2, 9));
 
-        // Changing routing status for remaining deployment adds back all deployments, because removing all deployments
+        // Changing routing status for remaining deployments adds back all deployments, because removing all deployments
         // puts all IN
         tester.routingPolicies().setRoutingStatus(betaZone1, RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
         betaContext.flushDnsUpdates();
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(betaZone1, 2,
-                        mainZone1, 8));
+                             Map.of(betaZone1, 2,
+                                    mainZone1, 8,
+                                    mainZone2, 9));
 
         // Activating main deployment allows us to deactivate the beta deployment
         tester.routingPolicies().setRoutingStatus(mainZone1, RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
         betaContext.flushDnsUpdates();
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
-                Map.of(mainZone1, 8));
+                             Map.of(mainZone1, 8));
 
         // Activate all deployments again
         tester.routingPolicies().setRoutingStatus(betaZone1, RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
         betaContext.flushDnsUpdates();
         tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
                 Map.of(betaZone1, 2,
-                        mainZone1, 8));
+                       mainZone1, 8,
+                       mainZone2, 9));
     }
 
     /** Returns an application package builder that satisfies requirements for a directly routed endpoint */
@@ -919,11 +935,17 @@ public class RoutingPoliciesTest {
             List<ZoneId> zones;
             if (tester.controller().system().isPublic()) {
                 zones = publicZones();
+                tester.controllerTester().setZones(zones);
             } else {
                 zones = new ArrayList<>(tester.controllerTester().zoneRegistry().zones().all().ids()); // Default zones
                 zones.add(zone4); // Missing from default ZoneRegistryMock zones
+                tester.controllerTester().setZones(zones);
+                tester.controllerTester().zoneRegistry().addZones(ZoneApiMock.newBuilder().withId(zone5.value()).withCloud("aws").build());
+                tester.controllerTester().zoneRegistry().addZones(ZoneApiMock.newBuilder().withId(zone6.value()).withCloud("gcp").build());
+                zones.add(zone5);
+                zones.add(zone6);
+                tester.configServer().bootstrap(zones, SystemApplication.notController());
             }
-            tester.controllerTester().setZones(zones);
             if (exclusiveRouting) {
                 tester.controllerTester().setRoutingMethod(zones, RoutingMethod.exclusive);
             }
@@ -997,7 +1019,7 @@ public class RoutingPoliciesTest {
                 deploymentsByDnsName.computeIfAbsent(dnsName, (k) -> new ArrayList<>())
                                     .add(deployment);
             }
-            assertEquals(1, deploymentsByDnsName.size(), "Found " + endpointId + " for " + application);
+            assertTrue(1 <= deploymentsByDnsName.size(), "Found " + endpointId + " for " + application);
             deploymentsByDnsName.forEach((dnsName, deployments) -> {
                 Set<String> weightedTargets = deployments.stream()
                                                            .map(d -> "weighted/lb-" + loadBalancerId + "--" +