Support duplicate regions within same global endpoint

To support duplicate regions within a global endpoint we create a combination of
latency and weighted alias targets.

In the following examples an application exists in `us-west-2`, `us-east-1a` and
`us-east-1b`. Before this change global endpoints pointed directly to the
zone endpoint:

```
(latency) ALIAS app1.tenant1.global.vespa.example.com -> app1.tenant1.us-west-2.vespa.example.com
(latency) ALIAS app1.tenant1.global.vespa.example.com -> app1.tenant1.us-east-1a.vespa.example.com
(latency) ALIAS app1.tenant1.global.vespa.example.com -> app1.tenant1.us-east-1b.vespa.example.com
```

After this change we introduce an additional level of names by creating a
weighted record per region:

```
(latency) ALIAS app1.tenant1.global.vespa.example.com -> app1.tenant1.us-west-2-w.vespa.example.com
  |- (weighted) ALIAS app1.tenant1.us-west-2-w.vespa.example.com -> app1.tenant1.us-west-2.vespa.example.com
(latency) ALIAS app1.tenant1.global.vespa.example.com -> app1.tenant1.us-east-1-w.vespa.example.com
  |- (weighted) ALIAS app1.tenant1.us-east-1-w.vespa.example.com -> app1.tenant1.us-east-1a.vespa.example.com
  |- (weighted) ALIAS app1.tenant1.us-east-1-w.vespa.example.com -> app1.tenant1.us-east-1b.vespa.example.com
```

Toggling global routing status now adjusts the weight (`0 = out`) instead of
removing records as this simplified the code.

7 files changed, 885 insertions, 106 deletions

diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java
index 7bd43ff1dcb..44acf1ab02e 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/LatencyAliasTarget.java
@@ -44,6 +44,11 @@ public class LatencyAliasTarget extends AliasTarget {
         return Objects.hash(super.hashCode(), zone);
     }
 
+    @Override
+    public String toString() {
+        return "latency target for " + name() + "[id=" + id() + ",dnsZone=" + dnsZone() + "]";
+    }
+
     /** Unpack latency alias from given record data */
     public static LatencyAliasTarget unpack(RecordData data) {
         var parts = data.asString().split("/");
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
index e8688b17347..03b10780e33 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/MemoryNameService.java
@@ -46,9 +46,10 @@ public class MemoryNameService implements NameService {
                              .map(d -> new Record(Record.Type.ALIAS, name, d.pack()))
                              .collect(Collectors.toList());
         // Satisfy idempotency contract of interface
-        records.stream()
-               .filter(r -> !this.records.contains(r))
-               .forEach(this::add);
+        for (var r1 : records) {
+            this.records.removeIf(r2 -> conflicts(r1, r2));
+        }
+        this.records.addAll(records);
         return records;
     }
 
@@ -115,10 +116,13 @@ public class MemoryNameService implements NameService {
      * most real name services.
      */
     private static boolean conflicts(Record r1, Record r2) {
-        if (!r1.name().equals(r2.name())) return false;               // Distinct names never conflict
-        if (r1.type() == Record.Type.ALIAS && r1.type() == r2.type()) // ALIAS records only require distinct data
-            return r1.data().equals(r2.data());
-        return true;                                                  // Anything else is considered a conflict
+        if (!r1.name().equals(r2.name())) return false;                // Distinct names never conflict
+        if (r1.type() == Record.Type.ALIAS && r1.type() == r2.type()) {
+            AliasTarget t1 = AliasTarget.unpack(r1.data());
+            AliasTarget t2 = AliasTarget.unpack(r2.data());
+            return t1.name().equals(t2.name());                        // ALIAS records require distinct targets
+        }
+        return true;                                                   // Anything else is considered a conflict
     }
 
 }
diff --git a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java
index 9d741cb2dbc..8f81c94257e 100644
--- a/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java
+++ b/controller-api/src/main/java/com/yahoo/vespa/hosted/controller/api/integration/dns/WeightedAliasTarget.java
@@ -47,6 +47,11 @@ public class WeightedAliasTarget extends AliasTarget {
         return Objects.hash(super.hashCode(), weight);
     }
 
+    @Override
+    public String toString() {
+        return "weighted target for " + name() + "[id=" + id() + ",dnsZone=" + dnsZone() + ",weight=" + weight + "]";
+    }
+
     /** Unpack weighted alias from given record data */
     public static WeightedAliasTarget unpack(RecordData data) {
         var parts = data.asString().split("/");
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index e080b7babce..306c2a1f1e2 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -3,9 +3,13 @@ package com.yahoo.vespa.hosted.controller.routing;
 
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.curator.Lock;
+import com.yahoo.vespa.flags.BooleanFlag;
+import com.yahoo.vespa.flags.FetchVector;
+import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
@@ -14,6 +18,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.LatencyAliasTarget;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder;
@@ -42,10 +48,12 @@ public class RoutingPolicies {
 
     private final Controller controller;
     private final CuratorDb db;
+    private final BooleanFlag weightedDnsPerRegion;
 
     public RoutingPolicies(Controller controller) {
         this.controller = Objects.requireNonNull(controller, "controller must be non-null");
         this.db = controller.curator();
+        this.weightedDnsPerRegion = Flags.WEIGHTED_DNS_PER_REGION.bindTo(controller.flagSource());
         try (var lock = db.lockRoutingPolicies()) { // Update serialized format
             for (var policy : db.readRoutingPolicies().entrySet()) {
                 db.writeRoutingPolicies(policy.getKey(), policy.getValue());
@@ -84,7 +92,7 @@ public class RoutingPolicies {
             removeGlobalDnsUnreferencedBy(allocation, lock);
             storePoliciesOf(allocation, lock);
             removePoliciesUnreferencedBy(allocation, lock);
-            updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock);
+            updateGlobalDnsOf(application, get(allocation.deployment.applicationId()).values(), inactiveZones, lock);
         }
     }
 
@@ -93,9 +101,9 @@ public class RoutingPolicies {
         try (var lock = db.lockRoutingPolicies()) {
             db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator,
                                                                                        controller.clock().instant())));
-            var allPolicies = db.readRoutingPolicies();
-            for (var applicationPolicies : allPolicies.values()) {
-                updateGlobalDnsOf(applicationPolicies.values(), Set.of(), lock);
+            Map<ApplicationId, Map<RoutingPolicyId, RoutingPolicy>> allPolicies = db.readRoutingPolicies();
+            for (var kv : allPolicies.entrySet()) {
+                updateGlobalDnsOf(kv.getKey(), kv.getValue().values(), Set.of(), lock);
             }
         }
     }
@@ -112,12 +120,12 @@ public class RoutingPolicies {
                 newPolicies.put(policy.id(), newPolicy);
             }
             db.writeRoutingPolicies(deployment.applicationId(), newPolicies);
-            updateGlobalDnsOf(newPolicies.values(), Set.of(), lock);
+            updateGlobalDnsOf(deployment.applicationId(), newPolicies.values(), Set.of(), lock);
         }
     }
 
     /** Update global DNS record for given policies */
-    private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
+    private void legacyUpdateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
         // Create DNS record for each routing ID
         var routingTable = routingTableFrom(routingPolicies);
         for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
@@ -156,6 +164,65 @@ public class RoutingPolicies {
         }
     }
 
+    // TODO(mpolden): Remove and inline call to updateGlobalDnsOf when feature flag disappears
+    private void updateGlobalDnsOf(ApplicationId application, Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
+        if (weightedDnsPerRegion.with(FetchVector.Dimension.APPLICATION_ID, application.serializedForm()).value()) {
+            updateGlobalDnsOf(routingPolicies, inactiveZones, lock);
+        } else {
+            legacyUpdateGlobalDnsOf(routingPolicies, inactiveZones, lock);
+        }
+    }
+
+    /** Update global DNS record for given policies */
+    private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
+        Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(routingPolicies);
+        for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
+            Map<LatencyAliasTarget, Set<AliasTarget>> targets = computeTargets(routeEntry.getValue(), inactiveZones);
+            // Create a weighted ALIAS per zone, pointing to all targets within that zone
+            targets.forEach(((latencyTarget, weightedTargets) -> {
+                controller.nameServiceForwarder().createAlias(RecordName.from(latencyTarget.name().value()),
+                                                              weightedTargets,
+                                                              Priority.normal);
+            }));
+            // Create global latency-based ALIAS pointing to each weighted ALIAS
+            var endpoints = controller.routing().endpointsOf(routeEntry.getKey().application())
+                                      .named(routeEntry.getKey().endpointId())
+                                      .not().requiresRotation();
+            Set<AliasTarget> latencyTargets = Collections.unmodifiableSet(targets.keySet());
+            endpoints.forEach(endpoint -> controller.nameServiceForwarder().createAlias(RecordName.from(endpoint.dnsName()),
+                                                                                        latencyTargets, Priority.normal));
+        }
+    }
+
+    /** Compute latency and associated weighted targets from given policies */
+    private Map<LatencyAliasTarget, Set<AliasTarget>> computeTargets(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) {
+        Map<LatencyAliasTarget, Set<AliasTarget>> targets = new LinkedHashMap<>();
+        RoutingMethod routingMethod = RoutingMethod.exclusive;
+        for (var policy : policies) {
+            if (policy.dnsZone().isEmpty()) continue;
+            if (!controller.zoneRegistry().routingMethods(policy.id().zone()).contains(routingMethod)) continue;
+            Endpoint weighted = policy.weightedEndpointIn(controller.system(), routingMethod);
+            LatencyAliasTarget latencyTarget = new LatencyAliasTarget(HostName.from(weighted.dnsName()),
+                                                                      policy.dnsZone().get(),
+                                                                      weighted.zones().get(0));
+            // Do not route to zone if global routing status is set out at:
+            // - zone level (ZoneRoutingPolicy)
+            // - deployment level (RoutingPolicy)
+            // - application package level (deployment.xml)
+            long weight = 1;
+            var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
+            if (isConfiguredOut(policy, zonePolicy, inactiveZones)) {
+                weight = 0; // A record with 0 weight will not received traffic. If all records within a group have 0
+                            // weight, traffic is routed to all records with equal probability.
+            }
+            var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(),
+                                                         policy.id().zone(), weight);
+            Set<AliasTarget> weightedTargets = targets.computeIfAbsent(latencyTarget, (k) -> new LinkedHashSet<>());
+            weightedTargets.add(weightedTarget);
+        }
+        return Collections.unmodifiableMap(targets);
+    }
+
     /** Store routing policies for given load balancers */
     private void storePoliciesOf(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) {
         var policies = new LinkedHashMap<>(get(allocation.deployment.applicationId()));
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java
new file mode 100644
index 00000000000..8479309c63b
--- /dev/null
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesLegacyTest.java
@@ -0,0 +1,712 @@
+// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.routing;
+
+import com.google.common.collect.Sets;
+import com.yahoo.config.application.api.DeploymentSpec;
+import com.yahoo.config.application.api.ValidationId;
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.AthenzDomain;
+import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.ClusterSpec;
+import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RegionName;
+import com.yahoo.config.provision.SystemName;
+import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.ControllerTester;
+import com.yahoo.vespa.hosted.controller.Instance;
+import com.yahoo.vespa.hosted.controller.RoutingController;
+import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
+import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
+import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
+import com.yahoo.vespa.hosted.controller.integration.ServiceRegistryMock;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
+import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
+import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
+import org.junit.Test;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.time.temporal.ChronoUnit;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * @author mortent
+ * @author mpolden
+ */
+// TODO(mpolden): Remove when weighted-dns-per-region flag is removed
+public class RoutingPoliciesLegacyTest {
+
+    private final ZoneId zone1 = ZoneId.from("prod", "us-west-1");
+    private final ZoneId zone2 = ZoneId.from("prod", "us-central-1");
+    private final ZoneId zone3 = ZoneId.from("prod", "us-east-3");
+
+    private final ApplicationPackage applicationPackage = applicationPackageBuilder().region(zone1.region())
+                                                                                     .region(zone2.region())
+                                                                                     .build();
+
+    @Test
+    public void global_routing_policies() {
+        var tester = new RoutingPoliciesTester();
+        var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
+        var context2 = tester.newDeploymentContext("tenant1", "app2", "default");
+        int clustersPerZone = 2;
+        int numberOfDeployments = 2;
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("r0", "c0")
+                .endpoint("r1", "c0", "us-west-1")
+                .endpoint("r2", "c1")
+                .build();
+        tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone1, zone2);
+
+        // Creates alias records
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1);
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2);
+        assertEquals("Routing policy count is equal to cluster count",
+                     numberOfDeployments * clustersPerZone,
+                     tester.policiesOf(context1.instance().id()).size());
+
+        // Applications gains a new deployment
+        ApplicationPackage applicationPackage2 = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .region(zone3.region())
+                .endpoint("r0", "c0")
+                .endpoint("r1", "c0", "us-west-1")
+                .endpoint("r2", "c1")
+                .build();
+        numberOfDeployments++;
+        tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), zone3);
+        context1.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+        // Endpoints are updated to contain cluster in new deployment
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0, zone1, zone2, zone3);
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0, zone1);
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 1, zone1, zone2, zone3);
+
+        // Another application is deployed with a single cluster and global endpoint
+        var endpoint4 = "r0.app2.tenant1.global.vespa.oath.cloud";
+        tester.provisionLoadBalancers(1, context2.instanceId(), zone1, zone2);
+        var applicationPackage3 = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("r0", "c0")
+                .build();
+        context2.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        tester.assertTargets(context2.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // All endpoints for app1 are removed
+        ApplicationPackage applicationPackage4 = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .region(zone3.region())
+                .allow(ValidationId.globalEndpointChange)
+                .build();
+        context1.submit(applicationPackage4).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r0"), 0);
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r1"), 0);
+        tester.assertTargets(context1.instanceId(), EndpointId.of("r2"), 0);
+        var policies = tester.policiesOf(context1.instanceId());
+        assertEquals(clustersPerZone * numberOfDeployments, policies.size());
+        assertTrue("Rotation membership is removed from all policies",
+                   policies.stream().allMatch(policy -> policy.endpoints().isEmpty()));
+        assertEquals("Rotations for " + context2.application() + " are not removed", 2, tester.aliasDataOf(endpoint4).size());
+    }
+
+    @Test
+    public void zone_routing_policies() {
+        zone_routing_policies(false);
+        zone_routing_policies(true);
+    }
+
+    private void zone_routing_policies(boolean sharedRoutingLayer) {
+        var tester = new RoutingPoliciesTester();
+        var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
+        var context2 = tester.newDeploymentContext("tenant1", "app2", "default");
+
+        // Deploy application
+        int clustersPerZone = 2;
+        tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2);
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+        // Deployment creates records and policies for all clusters in all zones
+        Set<String> expectedRecords = Set.of(
+                "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-central-1.vespa.oath.cloud"
+        );
+        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(4, tester.policiesOf(context1.instanceId()).size());
+
+        // Next deploy does nothing
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(4, tester.policiesOf(context1.instanceId()).size());
+
+        // Add 1 cluster in each zone and deploy
+        tester.provisionLoadBalancers(clustersPerZone + 1, context1.instanceId(), sharedRoutingLayer, zone1, zone2);
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        expectedRecords = Set.of(
+                "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c2.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c2.app1.tenant1.us-central-1.vespa.oath.cloud"
+        );
+        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(6, tester.policiesOf(context1.instanceId()).size());
+
+        // Deploy another application
+        tester.provisionLoadBalancers(clustersPerZone, context2.instanceId(), sharedRoutingLayer, zone1, zone2);
+        context2.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        expectedRecords = Set.of(
+                "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c2.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c2.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c0.app2.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app2.tenant1.us-central-1.vespa.oath.cloud",
+                "c0.app2.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app2.tenant1.us-west-1.vespa.oath.cloud"
+        );
+        assertEquals(expectedRecords.stream().sorted().collect(Collectors.toList()), tester.recordNames().stream().sorted().collect(Collectors.toList()));
+        assertEquals(4, tester.policiesOf(context2.instanceId()).size());
+
+        // Deploy removes cluster from app1
+        tester.provisionLoadBalancers(clustersPerZone, context1.instanceId(), sharedRoutingLayer, zone1, zone2);
+        context1.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        expectedRecords = Set.of(
+                "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c0.app2.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app2.tenant1.us-central-1.vespa.oath.cloud",
+                "c0.app2.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app2.tenant1.us-west-1.vespa.oath.cloud"
+        );
+        assertEquals(expectedRecords, tester.recordNames());
+
+        // Remove app2 completely
+        tester.controllerTester().controller().applications().requireInstance(context2.instanceId()).deployments().keySet()
+              .forEach(zone -> {
+                  tester.controllerTester().configServer().removeLoadBalancers(context2.instanceId(), zone);
+                  tester.controllerTester().controller().applications().deactivate(context2.instanceId(), zone);
+              });
+        context2.flushDnsUpdates();
+        expectedRecords = Set.of(
+                "c0.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-west-1.vespa.oath.cloud",
+                "c0.app1.tenant1.us-central-1.vespa.oath.cloud",
+                "c1.app1.tenant1.us-central-1.vespa.oath.cloud"
+        );
+        assertEquals(expectedRecords, tester.recordNames());
+        assertTrue("Removes stale routing policies " + context2.application(), tester.routingPolicies().get(context2.instanceId()).isEmpty());
+        assertEquals("Keeps routing policies for " + context1.application(), 4, tester.routingPolicies().get(context1.instanceId()).size());
+    }
+
+    @Test
+    public void global_routing_policies_in_rotationless_system() {
+        var tester = new RoutingPoliciesTester(new DeploymentTester(new ControllerTester(new RotationsConfig.Builder().build())));
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+        tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region().value())
+                .endpoint("r0", "c0")
+                .build();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+        var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud";
+        assertEquals(endpoint + " points to c0 in all regions",
+                     List.of("latency/lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
+                     tester.aliasDataOf(endpoint));
+        assertTrue("No rotations assigned", context.application().instances().values().stream()
+                                                   .map(Instance::rotations)
+                                                   .allMatch(List::isEmpty));
+    }
+
+    @Test
+    public void manual_deployment_creates_routing_policy() {
+        // Empty application package is valid in manually deployed environments
+        var tester = new RoutingPoliciesTester();
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+        var emptyApplicationPackage = new ApplicationPackageBuilder().build();
+        var zone = ZoneId.from("dev", "us-east-1");
+        var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
+        tester.controllerTester().serviceRegistry().zoneRegistry()
+              .setZones(zoneApi)
+              .exclusiveRoutingIn(zoneApi);
+
+        // Deploy to dev
+        tester.controllerTester().controller().applications().deploy(context.instanceId(), zone, Optional.of(emptyApplicationPackage), DeployOptions.none());
+        assertEquals("DeploymentSpec is not persisted", DeploymentSpec.empty, context.application().deploymentSpec());
+        context.flushDnsUpdates();
+
+        // Routing policy is created and DNS is updated
+        assertEquals(1, tester.policiesOf(context.instanceId()).size());
+        assertEquals(Set.of("app1.tenant1.us-east-1.dev.vespa.oath.cloud"), tester.recordNames());
+    }
+
+    @Test
+    public void manual_deployment_creates_routing_policy_with_non_empty_spec() {
+        // Initial deployment
+        var tester = new RoutingPoliciesTester();
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+        context.submit(applicationPackage).deploy();
+        var zone = ZoneId.from("dev", "us-east-1");
+        var zoneApi = ZoneApiMock.from(zone.environment(), zone.region());
+        tester.controllerTester().serviceRegistry().zoneRegistry()
+              .setZones(zoneApi)
+              .exclusiveRoutingIn(zoneApi);
+        var prodRecords = Set.of("app1.tenant1.us-central-1.vespa.oath.cloud", "app1.tenant1.us-west-1.vespa.oath.cloud");
+        assertEquals(prodRecords, tester.recordNames());
+
+        // Deploy to dev under different instance
+        var devInstance = context.application().id().instance("user");
+        tester.controllerTester().controller().applications().deploy(devInstance, zone, Optional.of(applicationPackage), DeployOptions.none());
+        assertEquals("DeploymentSpec is persisted", applicationPackage.deploymentSpec(), context.application().deploymentSpec());
+        context.flushDnsUpdates();
+
+        // Routing policy is created and DNS is updated
+        assertEquals(1, tester.policiesOf(devInstance).size());
+        assertEquals(Sets.union(prodRecords, Set.of("user.app1.tenant1.us-east-1.dev.vespa.oath.cloud")), tester.recordNames());
+    }
+
+    @Test
+    public void reprovisioning_load_balancer_preserves_cname_record() {
+        var tester = new RoutingPoliciesTester();
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+
+        // Initial load balancer is provisioned
+        tester.provisionLoadBalancers(1, context.instanceId(), zone1);
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region())
+                .build();
+
+        // Application is deployed
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        var expectedRecords = Set.of(
+                "c0.app1.tenant1.us-west-1.vespa.oath.cloud"
+        );
+        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(1, tester.policiesOf(context.instanceId()).size());
+
+        // Application is removed and the load balancer is deprovisioned
+        tester.controllerTester().controller().applications().deactivate(context.instanceId(), zone1);
+        tester.controllerTester().configServer().removeLoadBalancers(context.instanceId(), zone1);
+
+        // Load balancer for the same application is provisioned again, but with a different hostname
+        var newHostname = HostName.from("new-hostname");
+        var loadBalancer = new LoadBalancer("LB-0-Z-" + zone1.value(),
+                                            context.instanceId(),
+                                            ClusterSpec.Id.from("c0"),
+                                            newHostname,
+                                            LoadBalancer.State.active,
+                                            Optional.of("dns-zone-1"));
+        tester.controllerTester().configServer().putLoadBalancers(zone1, List.of(loadBalancer));
+
+        // Application redeployment preserves DNS record
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        assertEquals(expectedRecords, tester.recordNames());
+        assertEquals(1, tester.policiesOf(context.instanceId()).size());
+        assertEquals("CNAME points to current load blancer", newHostname.value() + ".",
+                     tester.cnameDataOf(expectedRecords.iterator().next()).get(0));
+    }
+
+    @Test
+    public void set_global_endpoint_status() {
+        var tester = new RoutingPoliciesTester();
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+
+        // Provision load balancers and deploy application
+        tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+                .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
+                .build();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+        // Global DNS record is created
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+
+        // Global routing status is overridden in one zone
+        var changedAt = tester.controllerTester().clock().instant();
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+
+        // Inactive zone is removed from global DNS record
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
+        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
+
+        // Status details is stored in policy
+        var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
+        assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status());
+        assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
+
+        // Other zone remains in
+        var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next();
+        assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status());
+        assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent());
+        assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt());
+
+        // Next deployment does not affect status
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
+        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
+
+        // Deployment is set back in
+        tester.controllerTester().clock().advance(Duration.ofHours(1));
+        changedAt = tester.controllerTester().clock().instant();
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+
+        policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
+        assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status());
+        assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
+
+        // Deployment is set out through a new deployment.xml
+        var applicationPackage2 = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region(), false)
+                .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+                .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
+                .build();
+        context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
+        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1);
+
+        // ... back in
+        var applicationPackage3 = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+                .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
+                .build();
+        context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+    }
+
+    @Test
+    public void set_zone_global_endpoint_status() {
+        var tester = new RoutingPoliciesTester();
+        var context1 = tester.newDeploymentContext("tenant1", "app1", "default");
+        var context2 = tester.newDeploymentContext("tenant2", "app2", "default");
+        var contexts = List.of(context1, context2);
+
+        // Deploy applications
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("default", "c0", zone1.region().value(), zone2.region().value())
+                .build();
+        for (var context : contexts) {
+            tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+            context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+            tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
+        }
+
+        // Set zone out
+        tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out);
+        context1.flushDnsUpdates();
+        tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
+        tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
+        for (var context : contexts) {
+            var policies = tester.routingPolicies().get(context.instanceId());
+            assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in,
+                       policies.values().stream()
+                               .map(RoutingPolicy::status)
+                               .map(Status::globalRouting)
+                               .map(GlobalRouting::status)
+                               .allMatch(status -> status == GlobalRouting.Status.in));
+        }
+        var changedAt = tester.controllerTester().clock().instant();
+        var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2);
+        assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status());
+        assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt());
+
+        // Setting status per deployment does not affect status as entire zone is out
+        tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        context1.flushDnsUpdates();
+        tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
+        tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
+
+        // Set single deployment out
+        tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
+        context1.flushDnsUpdates();
+
+        // Set zone back in. Deployment set explicitly out, remains out, the rest are in
+        tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in);
+        context1.flushDnsUpdates();
+        tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
+        tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
+    }
+
+    @Test
+    public void non_production_deployment_is_not_registered_in_global_endpoint() {
+        var tester = new RoutingPoliciesTester(SystemName.Public);
+
+        // Configure the system to use the same region for test, staging and prod
+        var sharedRegion = RegionName.from("aws-us-east-1c");
+        var prodZone = ZoneId.from(Environment.prod, sharedRegion);
+        var stagingZone = ZoneId.from(Environment.staging, sharedRegion);
+        var testZone = ZoneId.from(Environment.test, sharedRegion);
+        var zones = List.of(ZoneApiMock.from(prodZone),
+                            ZoneApiMock.from(stagingZone),
+                            ZoneApiMock.from(testZone));
+        tester.controllerTester().zoneRegistry()
+              .setZones(zones)
+              .setRoutingMethod(zones, RoutingMethod.exclusive);
+        tester.controllerTester().configServer().bootstrap(List.of(prodZone, stagingZone, testZone),
+                                                           SystemApplication.all());
+
+        var context = tester.tester.newDeploymentContext();
+        var endpointId = EndpointId.of("r0");
+        var applicationPackage = applicationPackageBuilder()
+                .trustDefaultCertificate()
+                .region(sharedRegion)
+                .endpoint(endpointId.id(), "default")
+                .build();
+
+        // Application starts deployment
+        context = context.submit(applicationPackage);
+        for (var testJob : List.of(JobType.systemTest, JobType.stagingTest)) {
+            context = context.runJob(testJob);
+            // Since runJob implicitly tears down the deployment and immediately deletes DNS records associated with the
+            // deployment, we consume only one DNS update at a time here
+            do {
+                context = context.flushDnsUpdates(1);
+                tester.assertTargets(context.instanceId(), endpointId, 0);
+            } while (!tester.recordNames().isEmpty());
+        }
+
+        // Deployment completes
+        context.completeRollout();
+        tester.assertTargets(context.instanceId(), endpointId, 0, prodZone);
+    }
+
+    @Test
+    public void changing_global_routing_status_never_removes_all_members() {
+        var tester = new RoutingPoliciesTester();
+        var context = tester.newDeploymentContext("tenant1", "app1", "default");
+
+        // Provision load balancers and deploy application
+        tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
+        var applicationPackage = applicationPackageBuilder()
+                .region(zone1.region())
+                .region(zone2.region())
+                .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
+                .build();
+        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
+
+        // Global DNS record is created, pointing to all configured zones
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // Global routing status is overridden for one deployment
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
+
+        // Setting other deployment out implicitly sets all deployments in
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // One inactive deployment is put back in. Global DNS record now points to the only active deployment
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
+
+        // Setting zone (containing active deployment) out puts all deployments in
+        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out);
+        context.flushDnsUpdates();
+        assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status());
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+
+        // Setting zone back in removes the currently inactive deployment
+        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in);
+        context.flushDnsUpdates();
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
+
+        // Inactive deployment is set in
+        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in,
+                                                        GlobalRouting.Agent.tenant);
+        context.flushDnsUpdates();
+        for (var policy : tester.routingPolicies().get(context.instanceId()).values()) {
+            assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status());
+        }
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+    }
+
+    @Test
+    public void config_server_routing_policy() {
+        var tester = new RoutingPoliciesTester();
+        var app = SystemApplication.configServer.id();
+        RecordName name = RecordName.from("cfg.prod.us-west-1.test.vip");
+        tester.controllerTester().nameService().add(new Record(Record.Type.A, name, RecordData.from("192.0.2.1")));
+
+        tester.provisionLoadBalancers(1, app, zone1);
+        tester.routingPolicies().refresh(app, DeploymentSpec.empty, zone1);
+        new NameServiceDispatcher(tester.tester.controller(), Duration.ofDays(1), Integer.MAX_VALUE).run();
+
+        List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.CNAME, name);
+        assertEquals(1, records.size());
+        assertEquals(RecordData.from("lb-0--hosted-vespa:zone-config-servers:default--prod.us-west-1."),
+                     records.get(0).data());
+    }
+
+    /** Returns an application package builder that satisfies requirements for a directly routed endpoint */
+    private static ApplicationPackageBuilder applicationPackageBuilder() {
+        return new ApplicationPackageBuilder()
+                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"))
+                .compileVersion(RoutingController.DIRECT_ROUTING_MIN_VERSION);
+    }
+
+    private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) {
+        List<LoadBalancer> loadBalancers = new ArrayList<>();
+        for (int i = 0; i < count; i++) {
+            HostName lbHostname;
+            if (shared) {
+                lbHostname = HostName.from("shared-lb--" + zone.value());
+            } else {
+                lbHostname = HostName.from("lb-" + i + "--" + application.serializedForm() +
+                                           "--" + zone.value());
+            }
+            loadBalancers.add(
+                    new LoadBalancer("LB-" + i + "-Z-" + zone.value(),
+                                     application,
+                                     ClusterSpec.Id.from("c" + i),
+                                     lbHostname,
+                                     LoadBalancer.State.active,
+                                     Optional.of("dns-zone-1")));
+        }
+        return loadBalancers;
+    }
+
+    private static class RoutingPoliciesTester {
+
+        private final DeploymentTester tester;
+
+        public RoutingPoliciesTester() {
+            this(SystemName.main);
+        }
+
+        public RoutingPoliciesTester(SystemName system) {
+            this(new DeploymentTester(new ControllerTester(new ServiceRegistryMock(system))));
+        }
+
+        public RoutingPolicies routingPolicies() {
+            return tester.controllerTester().controller().routing().policies();
+        }
+
+        public DeploymentContext newDeploymentContext(String tenant, String application, String instance) {
+            return tester.newDeploymentContext(tenant, application, instance);
+        }
+
+        public ControllerTester controllerTester() {
+            return tester.controllerTester();
+        }
+
+        public RoutingPoliciesTester(DeploymentTester tester) {
+            this.tester = tester;
+            // Make all zones directly routed
+            tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones());
+        }
+
+        private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) {
+            for (ZoneId zone : zones) {
+                tester.configServer().removeLoadBalancers(application, zone);
+                tester.configServer().putLoadBalancers(zone, createLoadBalancers(zone, application, shared, clustersPerZone));
+            }
+        }
+
+        private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, ZoneId... zones) {
+            provisionLoadBalancers(clustersPerZone, application, false, zones);
+        }
+
+        private Collection<RoutingPolicy> policiesOf(ApplicationId instance) {
+            return tester.controller().curator().readRoutingPolicies(instance).values();
+        }
+
+        private Set<String> recordNames() {
+            return tester.controllerTester().nameService().records().stream()
+                         .map(Record::name)
+                         .map(RecordName::asString)
+                         .collect(Collectors.toSet());
+        }
+
+        private List<String> aliasDataOf(String name) {
+            return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name)).stream()
+                         .map(Record::data)
+                         .map(RecordData::asString)
+                         .collect(Collectors.toList());
+        }
+
+        private List<String> cnameDataOf(String name) {
+            return tester.controllerTester().nameService().findRecords(Record.Type.CNAME, RecordName.from(name)).stream()
+                         .map(Record::data)
+                         .map(RecordData::asString)
+                         .collect(Collectors.toList());
+        }
+
+        private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) {
+            var endpoint = tester.controller().routing().endpointsOf(application)
+                                 .named(endpointId)
+                                 .targets(List.of(zone))
+                                 .primary()
+                                 .map(Endpoint::dnsName)
+                                 .orElse("<none>");
+            var zoneTargets = Arrays.stream(zone)
+                                    .map(z -> "latency/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" +
+                                              z.value() + "/dns-zone-1/" + z.value())
+                                    .collect(Collectors.toSet());
+            assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets,
+                         Set.copyOf(aliasDataOf(endpoint)));
+        }
+
+    }
+
+}
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 3b2fe95d0f7..14de468028e 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -14,15 +14,19 @@ import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.config.provision.zone.ZoneId;
+import com.yahoo.vespa.flags.Flags;
+import com.yahoo.vespa.flags.InMemoryFlagSource;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.Instance;
 import com.yahoo.vespa.hosted.controller.RoutingController;
 import com.yahoo.vespa.hosted.controller.api.application.v4.model.DeployOptions;
+import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
 import com.yahoo.vespa.hosted.controller.api.integration.deployment.JobType;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
@@ -40,15 +44,14 @@ import java.time.Duration;
 import java.time.Instant;
 import java.time.temporal.ChronoUnit;
 import java.util.ArrayList;
-import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertSame;
 import static org.junit.Assert.assertTrue;
 
 /**
@@ -245,10 +248,7 @@ public class RoutingPoliciesTest {
                 .build();
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
 
-        var endpoint = "r0.app1.tenant1.global.vespa.oath.cloud";
-        assertEquals(endpoint + " points to c0 in all regions",
-                     List.of("latency/lb-0--tenant1:app1:default--prod.us-west-1/dns-zone-1/prod.us-west-1"),
-                     tester.aliasDataOf(endpoint));
+        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
         assertTrue("No rotations assigned", context.application().instances().values().stream()
                                                    .map(Instance::rotations)
                                                    .allMatch(List::isEmpty));
@@ -367,9 +367,9 @@ public class RoutingPoliciesTest {
                                                         GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
 
-        // Inactive zone is removed from global DNS record
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
-        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
+        // Inactive zone is given zero weight
+        tester.assertWeight(0, context.instanceId(), 0, zone1);
+        tester.assertWeight(1, context.instanceId(), 0, zone2);
 
         // Status details is stored in policy
         var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
@@ -386,16 +386,15 @@ public class RoutingPoliciesTest {
         // Next deployment does not affect status
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
         context.flushDnsUpdates();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
-        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone2);
+        tester.assertWeight(0, context.instanceId(), 0, zone1);
+        tester.assertWeight(1, context.instanceId(), 0, zone2);
 
         // Deployment is set back in
         tester.controllerTester().clock().advance(Duration.ofHours(1));
         changedAt = tester.controllerTester().clock().instant();
         tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
-        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+        tester.assertWeight(1, context.instanceId(), 0, zone1, zone2);
 
         policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
         assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status());
@@ -410,8 +409,8 @@ public class RoutingPoliciesTest {
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
         context.submit(applicationPackage2).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
-        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1);
+        tester.assertWeight(1, context.instanceId(), 0, zone1);
+        tester.assertWeight(0, context.instanceId(), 0, zone2);
 
         // ... back in
         var applicationPackage3 = applicationPackageBuilder()
@@ -421,8 +420,7 @@ public class RoutingPoliciesTest {
                 .endpoint("r1", "c0", zone1.region().value(), zone2.region().value())
                 .build();
         context.submit(applicationPackage3).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
-        tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
+        tester.assertWeight(1, context.instanceId(), 0, zone1, zone2);
     }
 
     @Test
@@ -442,13 +440,14 @@ public class RoutingPoliciesTest {
             tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
             context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
             tester.assertTargets(context.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
+            tester.assertWeight(1, context.instanceId(), 0, zone1, zone2);
         }
 
         // Set zone out
         tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out);
         context1.flushDnsUpdates();
-        tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
-        tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
+        tester.assertWeight(1, context1.instanceId(), 0, zone1);
+        tester.assertWeight(0, context1.instanceId(), 0, zone2);
         for (var context : contexts) {
             var policies = tester.routingPolicies().get(context.instanceId());
             assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in,
@@ -467,8 +466,8 @@ public class RoutingPoliciesTest {
         // Setting status per deployment does not affect status as entire zone is out
         tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
         context1.flushDnsUpdates();
-        tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
-        tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
+        tester.assertWeight(0, context1.instanceId(), 0, zone2);
+        tester.assertWeight(0, context2.instanceId(), 0, zone2);
 
         // Set single deployment out
         tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
@@ -477,8 +476,9 @@ public class RoutingPoliciesTest {
         // Set zone back in. Deployment set explicitly out, remains out, the rest are in
         tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in);
         context1.flushDnsUpdates();
-        tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
-        tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
+        tester.assertWeight(1, context1.instanceId(), 0, zone1);
+        tester.assertWeight(0, context1.instanceId(), 0, zone2);
+        tester.assertWeight(1, context2.instanceId(), 0, zone1, zone2);
     }
 
     @Test
@@ -521,63 +521,7 @@ public class RoutingPoliciesTest {
 
         // Deployment completes
         context.completeRollout();
-        tester.assertTargets(context.instanceId(), endpointId, 0, prodZone);
-    }
-
-    @Test
-    public void changing_global_routing_status_never_removes_all_members() {
-        var tester = new RoutingPoliciesTester();
-        var context = tester.newDeploymentContext("tenant1", "app1", "default");
-
-        // Provision load balancers and deploy application
-        tester.provisionLoadBalancers(1, context.instanceId(), zone1, zone2);
-        var applicationPackage = applicationPackageBuilder()
-                .region(zone1.region())
-                .region(zone2.region())
-                .endpoint("r0", "c0", zone1.region().value(), zone2.region().value())
-                .build();
-        context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
-
-        // Global DNS record is created, pointing to all configured zones
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
-
-        // Global routing status is overridden for one deployment
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
-        context.flushDnsUpdates();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
-
-        // Setting other deployment out implicitly sets all deployments in
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
-        context.flushDnsUpdates();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
-
-        // One inactive deployment is put back in. Global DNS record now points to the only active deployment
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
-        context.flushDnsUpdates();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
-
-        // Setting zone (containing active deployment) out puts all deployments in
-        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out);
-        context.flushDnsUpdates();
-        assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status());
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
-
-        // Setting zone back in removes the currently inactive deployment
-        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in);
-        context.flushDnsUpdates();
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
-
-        // Inactive deployment is set in
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
-        context.flushDnsUpdates();
-        for (var policy : tester.routingPolicies().get(context.instanceId()).values()) {
-            assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status());
-        }
-        tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
+        tester.assertTargets(context.instanceId(), endpointId, ClusterSpec.Id.from("default"), 0, prodZone);
     }
 
     @Test
@@ -653,6 +597,7 @@ public class RoutingPoliciesTest {
             this.tester = tester;
             // Make all zones directly routed
             tester.controllerTester().zoneRegistry().exclusiveRoutingIn(tester.controllerTester().zoneRegistry().zones().all().zones());
+            ((InMemoryFlagSource) tester.controllerTester().controller().flagSource()).withBooleanFlag(Flags.WEIGHTED_DNS_PER_REGION.id(), true);
         }
 
         private void provisionLoadBalancers(int clustersPerZone, ApplicationId application, boolean shared, ZoneId... zones) {
@@ -691,19 +636,53 @@ public class RoutingPoliciesTest {
                          .collect(Collectors.toList());
         }
 
-        private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) {
-            var endpoint = tester.controller().routing().endpointsOf(application)
-                                 .named(endpointId)
-                                 .targets(List.of(zone))
-                                 .primary()
-                                 .map(Endpoint::dnsName)
-                                 .orElse("<none>");
-            var zoneTargets = Arrays.stream(zone)
-                                    .map(z -> "latency/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" +
-                                              z.value() + "/dns-zone-1/" + z.value())
-                                    .collect(Collectors.toSet());
-            assertEquals("Global endpoint " + endpoint + " points to expected zones", zoneTargets,
-                         Set.copyOf(aliasDataOf(endpoint)));
+        private void assertWeight(long expected, ApplicationId application, int loadBalancerId, ZoneId... zones) {
+            for (var zone : zones) {
+                Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone))
+                                          .scope(Endpoint.Scope.weighted)
+                                          .named(EndpointId.of("c" + loadBalancerId))
+                                          .asList()
+                                          .get(0);
+                List<Record> records = tester.controllerTester().nameService().findRecords(Record.Type.ALIAS,
+                                                                                           RecordName.from(weighted.dnsName()));
+                assertEquals(1, records.size());
+                assertEquals("Record " + weighted.dnsName() + " has expected weight",
+                             expected,
+                             WeightedAliasTarget.unpack(records.get(0).data())
+                                                .weight());
+            }
+        }
+
+        private void assertTargets(ApplicationId application, EndpointId endpointId, ClusterSpec.Id clusterId, int loadBalancerId, ZoneId... zones) {
+            Set<String> latencyTargets = new HashSet<>();
+            for (var zone : zones) {
+                Endpoint weighted = tester.controller().routing().endpointsOf(new DeploymentId(application, zone))
+                                          .scope(Endpoint.Scope.weighted)
+                                          .named(EndpointId.of(clusterId.value()))
+                                          .asList()
+                                          .get(0);
+                String latencyTarget = "latency/" + weighted.dnsName() + "/dns-zone-1/" +
+                                       weighted.zones().get(0).value();
+                String weightedTarget = "weighted/lb-" + loadBalancerId + "--" + application.serializedForm() + "--" +
+                                        zone.value() + "/dns-zone-1/" + zone.value() + "/1";
+                assertEquals("Weighted target points to load balancer",
+                             List.of(weightedTarget),
+                             aliasDataOf(weighted.dnsName()));
+                latencyTargets.add(latencyTarget);
+            }
+            String globalEndpoint = tester.controller().routing().endpointsOf(application)
+                                          .named(endpointId)
+                                          .targets(List.of(zones))
+                                          .primary()
+                                          .map(Endpoint::dnsName)
+                                          .orElse("<none>");
+            assertEquals("Global endpoint " + globalEndpoint + " points to expected weighted targets",
+                         latencyTargets, Set.copyOf(aliasDataOf(globalEndpoint)));
+
+        }
+
+        private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId... zones) {
+            assertTargets(application, endpointId, ClusterSpec.Id.from("c" + loadBalancerId), loadBalancerId, zones);
         }
 
     }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 64d020f4d46..9e27d0c4a0b 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -295,6 +295,13 @@ public class Flags {
             "Takes effect on next tick"
     );
 
+    public static final UnboundBooleanFlag WEIGHTED_DNS_PER_REGION = defineFeatureFlag(
+            "weighted-dns-per-region", false,
+            "Whether to create weighted DNS records per region in global endpoints",
+            "Takes effect on next deployment through controller",
+            APPLICATION_ID
+    );
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, String description,
                                                        String modificationEffect, FetchVector.Dimension... dimensions) {