Create global rotations from routing policies

7 files changed, 272 insertions, 60 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java
index 91406bdb941..0254bf2fd38 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/GlobalDnsName.java
@@ -2,9 +2,11 @@
 package com.yahoo.vespa.hosted.controller.application;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.RotationName;
 import com.yahoo.config.provision.SystemName;
 
 import java.net.URI;
+import java.util.Objects;
 
 /**
  * Represents an application's global rotation.
@@ -24,32 +26,36 @@ public class GlobalDnsName {
     private final URI oathUrl;
 
     public GlobalDnsName(ApplicationId application, SystemName system) {
-        this.url = URI.create(String.format("http://%s%s.%s.%s:%d/",
-                                            getSystemPart(system, "."),
+        this(application, system, null);
+    }
+
+    public GlobalDnsName(ApplicationId application, SystemName system, RotationName rotation) {
+        Objects.requireNonNull(application, "application must be non-null");
+        Objects.requireNonNull(system, "system must be non-null");
+
+        this.url = URI.create(String.format("http://%s%s%s.%s.%s:%d/",
+                                            clusterPart(rotation, "."),
+                                            systemPart(system, "."),
                                             sanitize(application.application().value()),
                                             sanitize(application.tenant().value()),
                                             DNS_SUFFIX,
                                             port));
-        this.secureUrl = URI.create(String.format("https://%s%s--%s.%s:%d/",
-                                                  getSystemPart(system, "--"),
+        this.secureUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/",
+                                                  clusterPart(rotation, "--"),
+                                                  systemPart(system, "--"),
                                                   sanitize(application.application().value()),
                                                   sanitize(application.tenant().value()),
                                                   DNS_SUFFIX,
                                                   securePort));
-        this.oathUrl = URI.create(String.format("https://%s%s--%s.%s:%d/",
-                                                getSystemPart(system, "--"),
+        this.oathUrl = URI.create(String.format("https://%s%s%s--%s.%s:%d/",
+                                                clusterPart(rotation, "--"),
+                                                systemPart(system, "--"),
                                                 sanitize(application.application().value()),
                                                 sanitize(application.tenant().value()),
                                                 OATH_DNS_SUFFIX,
                                                 securePort));
     }
 
-    private String getSystemPart(SystemName system, String separator) {
-        return SystemName.main.equals(system)
-                ? ""
-                : system.name() + separator;
-    }
-
     /** URL to this rotation */
     public URI url() {
         return url;
@@ -85,4 +91,12 @@ public class GlobalDnsName {
         return s.replace('_', '-');
     }
 
+    private static String clusterPart(RotationName rotation, String separator) {
+        return rotation == null ? "" : rotation.value() + separator;
+    }
+
+    private static String systemPart(SystemName system, String separator) {
+        return SystemName.main == system ? "" : system.name() + separator;
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java
new file mode 100644
index 00000000000..c9378e27b61
--- /dev/null
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingId.java
@@ -0,0 +1,46 @@
+// Copyright 2019 Oath Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.vespa.hosted.controller.application;
+
+import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.RotationName;
+
+import java.util.Objects;
+
+/**
+ * Unique identifier for a global routing table entry (application x rotation name).
+ *
+ * @author mpolden
+ */
+public class RoutingId {
+
+    private final ApplicationId application;
+    private final RotationName rotation;
+
+    public RoutingId(ApplicationId application, RotationName rotation) {
+        this.application = Objects.requireNonNull(application, "application must be non-null");
+        this.rotation = Objects.requireNonNull(rotation, "rotation must be non-null");
+    }
+
+    public ApplicationId application() {
+        return application;
+    }
+
+    public RotationName rotation() {
+        return rotation;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) return true;
+        if (o == null || getClass() != o.getClass()) return false;
+        RoutingId that = (RoutingId) o;
+        return application.equals(that.application) &&
+               rotation.equals(that.rotation);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(application, rotation);
+    }
+
+}
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java
index 41e3a6086d2..9924727edbc 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/RoutingPolicy.java
@@ -20,21 +20,24 @@ import java.util.stream.Collectors;
  * Represents the DNS routing policy for a load balancer.
  *
  * @author mortent
+ * @author mpolden
  */
 public class RoutingPolicy {
 
     private static final String ignoredEndpointPart = "default";
 
     private final ApplicationId owner;
+    private final ZoneId zone;
     private final String recordId;
     private final HostName alias;
     private final HostName canonicalName;
     private final Optional<String> dnsZone;
     private final Set<RotationName> rotations;
 
-    public RoutingPolicy(ApplicationId owner, String recordId, HostName alias, HostName canonicalName,
+    public RoutingPolicy(ApplicationId owner, ZoneId zone, String recordId, HostName alias, HostName canonicalName,
                          Optional<String> dnsZone, Set<RotationName> rotations) {
         this.owner = Objects.requireNonNull(owner, "owner must be non-null");
+        this.zone = Objects.requireNonNull(zone, "zone must be non-null");
         this.recordId = Objects.requireNonNull(recordId, "recordId must be non-null");
         this.alias = Objects.requireNonNull(alias, "alias must be non-null");
         this.canonicalName = Objects.requireNonNull(canonicalName, "canonicalName must be non-null");
@@ -47,17 +50,22 @@ public class RoutingPolicy {
         return owner;
     }
 
+    /** The zone this applies to */
+    public ZoneId zone() {
+        return zone;
+    }
+
     /** The ID of the DNS record identifying this */
     public String recordId() {
         return recordId;
     }
 
-    /** This alias (lhs of a CNAME record) */
+    /** This alias (lhs of a CNAME or ALIAS record) */
     public HostName alias() {
         return alias;
     }
 
-    /** The canonical name for this (rhs of a CNAME record) */
+    /** The canonical name for this (rhs of a CNAME or ALIAS record) */
     public HostName canonicalName() {
         return canonicalName;
     }
@@ -78,6 +86,7 @@ public class RoutingPolicy {
         if (o == null || getClass() != o.getClass()) return false;
         RoutingPolicy that = (RoutingPolicy) o;
         return owner.equals(that.owner) &&
+               zone.equals(that.zone) &&
                recordId.equals(that.recordId) &&
                alias.equals(that.alias) &&
                canonicalName.equals(that.canonicalName) &&
@@ -87,13 +96,14 @@ public class RoutingPolicy {
 
     @Override
     public int hashCode() {
-        return Objects.hash(owner, recordId, alias, canonicalName, dnsZone, rotations);
+        return Objects.hash(owner, zone, recordId, alias, canonicalName, dnsZone, rotations);
     }
 
     @Override
     public String toString() {
-        return String.format("%s: %s -> %s [rotations: %s%s], owned by %s", recordId, alias, canonicalName, rotations,
-                             dnsZone.map(z -> ", DNS zone: " + z).orElse(""), owner.toShortString());
+        return String.format("%s: %s -> %s [rotations: %s%s], owned by %s, in %s", recordId, alias, canonicalName, rotations,
+                             dnsZone.map(z -> ", DNS zone: " + z).orElse(""), owner.toShortString(),
+                             zone.value());
     }
 
     public static String createAlias(ClusterSpec.Id clusterId, ApplicationId applicationId, ZoneId zoneId) {
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java
index bc684e753d1..c778b103700 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainer.java
@@ -3,17 +3,21 @@ package com.yahoo.vespa.hosted.controller.maintenance;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RotationName;
 import com.yahoo.log.LogLevel;
 import com.yahoo.vespa.curator.Lock;
 import com.yahoo.vespa.hosted.controller.Controller;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 import com.yahoo.vespa.hosted.controller.api.integration.configserver.LoadBalancer;
+import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordId;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
+import com.yahoo.vespa.hosted.controller.application.GlobalDnsName;
+import com.yahoo.vespa.hosted.controller.application.RoutingId;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.persistence.CuratorDb;
 
@@ -56,15 +60,17 @@ public class RoutingPolicyMaintainer extends Maintainer {
     @Override
     protected void maintain() {
         Map<DeploymentId, List<LoadBalancer>> loadBalancers = findLoadBalancers();
-        updateDnsRecords(loadBalancers);
-        removeObsoleteDnsRecords(loadBalancers);
+        registerCnames(loadBalancers);
+        removeObsoleteCnames(loadBalancers);
+        registerAliases();
+        removeObsoleteAliases(loadBalancers);
     }
 
-    /** Find all exclusive load balancers owned by given applications, grouped by deployment */
+    /** Find all exclusive load balancers in this system, grouped by deployment */
     private Map<DeploymentId, List<LoadBalancer>> findLoadBalancers() {
         Map<DeploymentId, List<LoadBalancer>> result = new LinkedHashMap<>();
         for (ZoneId zone : controller().zoneRegistry().zones().controllerUpgraded().ids()) {
-            List<LoadBalancer> loadBalancers = findLoadBalancersIn(zone);
+            List<LoadBalancer> loadBalancers = controller().applications().configServer().getLoadBalancers(zone);
             for (LoadBalancer loadBalancer : loadBalancers) {
                 DeploymentId deployment = new DeploymentId(loadBalancer.application(), zone);
                 result.compute(deployment, (k, existing) -> {
@@ -79,8 +85,33 @@ public class RoutingPolicyMaintainer extends Maintainer {
         return Collections.unmodifiableMap(result);
     }
 
-    /** Create DNS records for all exclusive load balancers */
-    private void updateDnsRecords(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
+    /** Create aliases (global rotations) for all current routing policies */
+    private void registerAliases() {
+        try (Lock lock = db.lockRoutingPolicies()) {
+            Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(db.readRoutingPolicies());
+
+            // Create DNS record for each routing ID
+            for (Map.Entry<RoutingId, List<RoutingPolicy>> route : routingTable.entrySet()) {
+                GlobalDnsName dnsName = dnsName(route.getKey());
+                Set<AliasTarget> targets = route.getValue()
+                                                .stream()
+                                                .filter(policy -> policy.dnsZone().isPresent())
+                                                .map(policy -> new AliasTarget(policy.alias(),
+                                                                               policy.dnsZone().get(),
+                                                                               policy.zone()))
+                                                .collect(Collectors.toSet());
+                try {
+                    nameService.createAlias(RecordName.from(dnsName.oathDnsName()), targets);
+                } catch (Exception e) {
+                    log.log(LogLevel.WARNING, "Failed to create or update DNS record for global rotation " +
+                                              dnsName.oathDnsName() + ". Retrying in " + maintenanceInterval(), e);
+                }
+            }
+        }
+    }
+
+    /** Create CNAME records for each individual load balancers */
+    private void registerCnames(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
         for (Map.Entry<DeploymentId, List<LoadBalancer>> entry : loadBalancers.entrySet()) {
             ApplicationId application = entry.getKey().applicationId();
             ZoneId zone = entry.getKey().zoneId();
@@ -88,7 +119,7 @@ public class RoutingPolicyMaintainer extends Maintainer {
                 Set<RoutingPolicy> policies = new LinkedHashSet<>(db.readRoutingPolicies(application));
                 for (LoadBalancer loadBalancer : entry.getValue()) {
                     try {
-                        policies.add(registerDnsAlias(application, zone, loadBalancer));
+                        policies.add(registerCname(application, zone, loadBalancer));
                     } catch (Exception e) {
                         log.log(LogLevel.WARNING, "Failed to create or update DNS record for load balancer " +
                                                   loadBalancer.hostname() + ". Retrying in " + maintenanceInterval(),
@@ -101,7 +132,7 @@ public class RoutingPolicyMaintainer extends Maintainer {
     }
 
     /** Register DNS alias for given load balancer */
-    private RoutingPolicy registerDnsAlias(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) {
+    private RoutingPolicy registerCname(ApplicationId application, ZoneId zone, LoadBalancer loadBalancer) {
         HostName alias = HostName.from(RoutingPolicy.createAlias(loadBalancer.cluster(), application, zone));
         RecordName name = RecordName.from(alias.value());
         RecordData data = RecordData.fqdn(loadBalancer.hostname().value());
@@ -113,28 +144,18 @@ public class RoutingPolicyMaintainer extends Maintainer {
         RecordId id;
         if (record.isPresent()) {
             id = record.get().id();
-            nameService.updateRecord(id, data);
+            if (!record.get().data().equals(data)) {
+                nameService.updateRecord(id, data);
+            }
         } else {
             id = nameService.createCname(name, data);
         }
-        return new RoutingPolicy(application, id.asString(), alias, loadBalancer.hostname(), loadBalancer.dnsZone(),
-                                 loadBalancer.rotations());
-    }
-
-    /** Find all load balancers in given zone */
-    private List<LoadBalancer> findLoadBalancersIn(ZoneId zone) {
-        try {
-            return controller().applications().configServer().getLoadBalancers(zone);
-        } catch (Exception e) {
-            log.log(LogLevel.WARNING,
-                    String.format("Got exception fetching load balancers in zone: %s. Retrying in %s",
-                                  zone.value(), maintenanceInterval()),  e);
-        }
-        return Collections.emptyList();
+        return new RoutingPolicy(application, zone, id.asString(), alias, loadBalancer.hostname(),
+                                 loadBalancer.dnsZone(), loadBalancer.rotations());
     }
 
     /** Remove all DNS records that point to non-existing load balancers */
-    private void removeObsoleteDnsRecords(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
+    private void removeObsoleteCnames(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
         try (Lock lock = db.lockRoutingPolicies()) {
             List<RoutingPolicy> removalCandidates = new ArrayList<>(db.readRoutingPolicies());
             Set<HostName> activeLoadBalancers = loadBalancers.values().stream()
@@ -143,16 +164,71 @@ public class RoutingPolicyMaintainer extends Maintainer {
                                                              .collect(Collectors.toSet());
 
             // Remove any active load balancers
-            removalCandidates.removeIf(lb -> activeLoadBalancers.contains(lb.canonicalName()));
+            removalCandidates.removeIf(policy -> activeLoadBalancers.contains(policy.canonicalName()));
             for (RoutingPolicy policy : removalCandidates) {
                 try {
                     nameService.removeRecord(new RecordId(policy.recordId()));
                 } catch (Exception e) {
-                    log.log(LogLevel.WARNING, "Failed to remove DNS record with ID '" + policy.recordId() +
+                    log.log(LogLevel.WARNING, "Failed to remove CNAME record with ID '" + policy.recordId() +
                                               "'. Retrying in " + maintenanceInterval());
                 }
             }
         }
     }
 
+    /** Remove global rotations that are not referenced by given load balancers */
+    private void removeObsoleteAliases(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
+        try (Lock lock = db.lockRoutingPolicies()) {
+            Set<RoutingId> removalCandidates = routingTableFrom(db.readRoutingPolicies()).keySet();
+            Set<RoutingId> activeRoutingIds = routingIdsFrom(loadBalancers);
+            removalCandidates.removeAll(activeRoutingIds);
+            for (RoutingId id : removalCandidates) {
+                GlobalDnsName dnsName = dnsName(id);
+                try {
+                    List<Record> records = nameService.findRecords(Record.Type.ALIAS, RecordName.from(dnsName.oathDnsName()));
+                    records.stream().map(Record::id).forEach(nameService::removeRecord);
+                } catch (Exception e) {
+                    log.log(LogLevel.WARNING, "Failed to remove all ALIAS records with name '" + dnsName.oathDnsName() +
+                                              "'. Retrying in " + maintenanceInterval());
+                }
+            }
+        }
+    }
+
+    /** Create a global DNS name for given routing ID */
+    private GlobalDnsName dnsName(RoutingId routingId) {
+        return new GlobalDnsName(routingId.application(), controller().system(), routingId.rotation());
+    }
+
+    /** Compute routing IDs from given load balancers */
+    private static Set<RoutingId> routingIdsFrom(Map<DeploymentId, List<LoadBalancer>> loadBalancers) {
+        Set<RoutingId> routingIds = new LinkedHashSet<>();
+        for (List<LoadBalancer> values : loadBalancers.values()) {
+            for (LoadBalancer loadBalancer : values) {
+                for (RotationName rotation : loadBalancer.rotations()) {
+                    routingIds.add(new RoutingId(loadBalancer.application(), rotation));
+                }
+            }
+        }
+        return Collections.unmodifiableSet(routingIds);
+    }
+
+    /** Compute a routing table from given policies. */
+    private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Set<RoutingPolicy> routingPolicies) {
+        Map<RoutingId, List<RoutingPolicy>> routingTable = new LinkedHashMap<>();
+        for (RoutingPolicy policy : routingPolicies) {
+            for (RotationName rotation : policy.rotations()) {
+                RoutingId id = new RoutingId(policy.owner(), rotation);
+                routingTable.compute(id, (k, policies) -> {
+                    if (policies == null) {
+                        policies = new ArrayList<>();
+                    }
+                    policies.add(policy);
+                    return policies;
+                });
+            }
+        }
+        return routingTable;
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
index 479d87cbe71..ccea83caef4 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializer.java
@@ -2,12 +2,15 @@
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.RotationName;
 import com.yahoo.slime.ArrayTraverser;
 import com.yahoo.slime.Cursor;
 import com.yahoo.slime.Inspector;
 import com.yahoo.slime.Slime;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 
 import java.util.Collections;
@@ -29,6 +32,7 @@ public class RoutingPolicySerializer {
     private static final String recordIdField = "recordId";
     private static final String aliasField = "alias";
     private static final String canonicalNameField = "canonicalName";
+    private static final String zoneField = "zone";
     private static final String dnsZoneField = "dnsZone";
     private static final String rotationsField = "rotations";
 
@@ -40,6 +44,7 @@ public class RoutingPolicySerializer {
             Cursor policyObject = policyArray.addObject();
             policyObject.setString(recordIdField, policy.recordId());
             policyObject.setString(aliasField, policy.alias().value());
+            policyObject.setString(zoneField, policy.zone().value());
             policyObject.setString(canonicalNameField, policy.canonicalName().value());
             policy.dnsZone().ifPresent(dnsZone -> policyObject.setString(dnsZoneField, dnsZone));
             Cursor rotationArray = policyObject.setArray(rotationsField);
@@ -65,6 +70,8 @@ public class RoutingPolicySerializer {
                 recordId = inspect.field(idField); // TODO: Remove after 7.9 has been released
             }
             policies.add(new RoutingPolicy(owner,
+                                           // TODO: Remove fallback after 7.13 has been released
+                                           optionalField(inspect.field(zoneField), ZoneId::from).orElse(ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName())),
                                            recordId.asString(),
                                            HostName.from(inspect.field(aliasField).asString()),
                                            HostName.from(inspect.field(canonicalNameField).asString()),
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java
index 5bd5bdb4983..ce691bf4de0 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/maintenance/RoutingPolicyMaintainerTest.java
@@ -14,6 +14,7 @@ import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
+import com.yahoo.vespa.hosted.controller.deployment.BuildJob;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
 import com.yahoo.vespa.hosted.controller.persistence.MockCuratorDb;
 import org.junit.Test;
@@ -26,6 +27,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.function.Supplier;
 import java.util.stream.Collectors;
 
 import static org.junit.Assert.assertEquals;
@@ -37,6 +39,8 @@ public class RoutingPolicyMaintainerTest {
 
     private final DeploymentTester tester = new DeploymentTester();
     private final Application app1 = tester.createApplication("app1", "tenant1", 1, 1L);
+    private final Application app2 = tester.createApplication("app2", "tenant1", 1, 1L);
+
     private final RoutingPolicyMaintainer maintainer = new RoutingPolicyMaintainer(tester.controller(), Duration.ofHours(12),
                                                                                    new JobControl(new MockCuratorDb()),
                                                                                    tester.controllerTester().nameService(),
@@ -48,6 +52,53 @@ public class RoutingPolicyMaintainerTest {
             .build();
 
     @Test
+    public void maintains_global_routing_policies() {
+        int clustersPerZone = 2;
+        tester.deployCompletely(app1, applicationPackage);
+        Map<Integer, Set<RotationName>> rotations = Map.of(0, Set.of(RotationName.from("r0")));
+        provisionLoadBalancers(app1, clustersPerZone, rotations);
+
+        // Creates alias record for cluster0
+        maintainer.maintain();
+        Supplier<List<Record>> records1 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app1--tenant1.global.vespa.oath.cloud"));
+        assertEquals(2, records1.get().size());
+        assertEquals("c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud.", records1.get().get(0).data().asString());
+        assertEquals("c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud.", records1.get().get(1).data().asString());
+
+        // Applications gains a new deployment
+        ApplicationPackage updatedApplicationPackage = new ApplicationPackageBuilder()
+                .environment(Environment.prod)
+                .region("us-west-1")
+                .region("us-central-1")
+                .region("us-east-3")
+                .build();
+        tester.deployCompletely(app1, updatedApplicationPackage, BuildJob.defaultBuildNumber + 1);
+
+        // Cluster in new deployment is added to the rotation
+        provisionLoadBalancers(app1, 2, rotations);
+        maintainer.maintain();
+        assertEquals(3, records1.get().size());
+        assertEquals("c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud.", records1.get().get(0).data().asString());
+        assertEquals("c0--app1--tenant1.prod.us-east-3.vespa.oath.cloud.", records1.get().get(1).data().asString());
+        assertEquals("c0--app1--tenant1.prod.us-west-1.vespa.oath.cloud.", records1.get().get(2).data().asString());
+
+        // Another appplication is deployed
+        Supplier<List<Record>> records2 = () -> tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from("r0--app2--tenant1.global.vespa.oath.cloud"));
+        tester.deployCompletely(app2, applicationPackage);
+        provisionLoadBalancers(app2, 1, Map.of(0, Set.of(RotationName.from("r0"))));
+        maintainer.maintain();
+        assertEquals(2, records2.get().size());
+        assertEquals("c0--app2--tenant1.prod.us-central-1.vespa.oath.cloud.", records2.get().get(0).data().asString());
+        assertEquals("c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud.", records2.get().get(1).data().asString());
+
+        // Rotation for app1 is removed
+        provisionLoadBalancers(app1, clustersPerZone, Collections.emptyMap());
+        maintainer.maintain();
+        assertEquals(0, records1.get().size());
+        assertEquals("Rotations for " + app2 + " are not removed", 2, records2.get().size());
+    }
+
+    @Test
     public void maintains_routing_policies_per_zone() {
         // Deploy application
         int clustersPerZone = 2;
@@ -62,12 +113,12 @@ public class RoutingPolicyMaintainerTest {
                 "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud",
                 "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, records());
+        assertEquals(expectedRecords, recordNames());
         assertEquals(4, policies(app1).size());
 
         // Next run does nothing
         maintainer.maintain();
-        assertEquals(expectedRecords, records());
+        assertEquals(expectedRecords, recordNames());
         assertEquals(4, policies(app1).size());
 
         // Add 1 cluster in each zone
@@ -81,11 +132,10 @@ public class RoutingPolicyMaintainerTest {
                 "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud",
                 "c2--app1--tenant1.prod.us-central-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, records());
+        assertEquals(expectedRecords, recordNames());
         assertEquals(6, policies(app1).size());
 
         // Add another application
-        Application app2 = tester.createApplication("app2", "tenant1", 1, 1L);
         tester.deployCompletely(app2, applicationPackage);
         provisionLoadBalancers(app2, clustersPerZone);
         maintainer.maintain();
@@ -101,10 +151,9 @@ public class RoutingPolicyMaintainerTest {
                 "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud",
                 "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, records());
+        assertEquals(expectedRecords, recordNames());
         assertEquals(4, policies(app2).size());
 
-
         // Remove cluster from app1
         provisionLoadBalancers(app1, clustersPerZone);
         maintainer.maintain();
@@ -118,7 +167,7 @@ public class RoutingPolicyMaintainerTest {
                 "c0--app2--tenant1.prod.us-west-1.vespa.oath.cloud",
                 "c1--app2--tenant1.prod.us-west-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, records());
+        assertEquals(expectedRecords, recordNames());
 
         // Remove app2 completely
         tester.controller().applications().require(app2.id()).deployments().keySet()
@@ -133,14 +182,18 @@ public class RoutingPolicyMaintainerTest {
                 "c0--app1--tenant1.prod.us-central-1.vespa.oath.cloud",
                 "c1--app1--tenant1.prod.us-central-1.vespa.oath.cloud"
         );
-        assertEquals(expectedRecords, records());
+        assertEquals(expectedRecords, recordNames());
     }
 
     private Set<RoutingPolicy> policies(Application application) {
         return tester.controller().curator().readRoutingPolicies(application.id());
     }
 
-    private Set<String> records() {
+    private List<Record> findAlias(String name) {
+        return tester.controllerTester().nameService().findRecords(Record.Type.ALIAS, RecordName.from(name));
+    }
+
+    private Set<String> recordNames() {
         return tester.controllerTester().nameService().records().values().stream()
                      .flatMap(Collection::stream)
                      .map(Record::name)
@@ -148,15 +201,18 @@ public class RoutingPolicyMaintainerTest {
                      .collect(Collectors.toSet());
     }
 
-    private void provisionLoadBalancers(Application application, int numberOfClustersPerZone) {
+    private void provisionLoadBalancers(Application application, int clustersPerZone, Map<Integer, Set<RotationName>> clusterRotations) {
         tester.controller().applications().require(application.id())
               .deployments().keySet()
               .forEach(zone -> tester.configServer().removeLoadBalancers(application.id(), zone));
         tester.controller().applications().require(application.id())
               .deployments().keySet()
               .forEach(zone -> tester.configServer()
-                                     .addLoadBalancers(zone, createLoadBalancers(zone, application.id(), numberOfClustersPerZone)));
+                                     .addLoadBalancers(zone, createLoadBalancers(zone, application.id(), clustersPerZone, clusterRotations)));
+    }
 
+    private void provisionLoadBalancers(Application application, int clustersPerZone) {
+        provisionLoadBalancers(application, clustersPerZone, Collections.emptyMap());
     }
 
     private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count,
@@ -176,8 +232,4 @@ public class RoutingPolicyMaintainerTest {
         return loadBalancers;
     }
 
-    private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, int count) {
-        return createLoadBalancers(zone, application, count, Collections.emptyMap());
-    }
-
 }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
index 7dc31581d3b..99a2f6406ef 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
@@ -3,9 +3,12 @@ package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.provision.ApplicationId;
+import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.HostName;
+import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.RotationName;
 import com.yahoo.vespa.config.SlimeUtils;
+import com.yahoo.vespa.hosted.controller.api.integration.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.RoutingPolicy;
 import org.junit.Test;
 
@@ -26,12 +29,14 @@ public class RoutingPolicySerializerTest {
         ApplicationId owner = ApplicationId.defaultId();
         Set<RotationName> rotations = Set.of(RotationName.from("r1"), RotationName.from("r2"));
         Set<RoutingPolicy> loadBalancers = ImmutableSet.of(new RoutingPolicy(owner,
+                                                                             ZoneId.from("prod", "us-north-1"),
                                                                              "record-id-1",
                                                                              HostName.from("my-pretty-alias"),
                                                                              HostName.from("long-and-ugly-name"),
                                                                              Optional.of("zone1"),
                                                                              rotations),
                                                            new RoutingPolicy(owner,
+                                                                             ZoneId.from("prod", "us-north-2"),
                                                                              "record-id-2",
                                                                              HostName.from("my-pretty-alias-2"),
                                                                              HostName.from("long-and-ugly-name-2"),
@@ -41,7 +46,7 @@ public class RoutingPolicySerializerTest {
         assertEquals(loadBalancers, serialized);
     }
 
-    // TODO: Remove after 7.9 has been released
+    // TODO: Remove after 7.10 has been released
     @Test
     public void test_serialization_old_format() {
         String json = "{\n" +
@@ -60,12 +65,14 @@ public class RoutingPolicySerializerTest {
                       "}\n";
         ApplicationId owner = ApplicationId.defaultId();
         Set<RoutingPolicy> loadBalancers = ImmutableSet.of(new RoutingPolicy(owner,
+                                                                             ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()),
                                                                              "record-id-1",
                                                                              HostName.from("my-pretty-alias"),
                                                                              HostName.from("long-and-ugly-name"),
                                                                              Optional.empty(),
                                                                              Collections.emptySet()),
                                                            new RoutingPolicy(owner,
+                                                                             ZoneId.from(Environment.defaultEnvironment(), RegionName.defaultName()),
                                                                              "record-id-2",
                                                                              HostName.from("my-pretty-alias-2"),
                                                                              HostName.from("long-and-ugly-name-2"),