Maintain ALIAS records for application-level endpoints

10 files changed, 426 insertions, 94 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 6be62367407..5d02302795d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -139,6 +139,29 @@ public class RoutingController {
         return EndpointList.copyOf(endpoints);
     }
 
+    /** Returns application-scoped endpoints for given application */
+    public EndpointList endpointsOf(TenantAndApplicationId applicationId) {
+        Application app = controller.applications().requireApplication(applicationId);
+        Set<Endpoint> endpoints = new LinkedHashSet<>();
+        for (var declaredEndpoint : app.deploymentSpec().endpoints()) {
+            Map<DeploymentId, Integer> deployments = declaredEndpoint.targets().stream()
+                                                                     .collect(Collectors.toMap(t -> new DeploymentId(applicationId.instance(t.instance()),
+                                                                                                                     ZoneId.from(Environment.prod, t.region())),
+                                                                                               t -> t.weight()));
+            List<RoutingMethod> availableRoutingMethods = routingMethodsOfAll(deployments.keySet(), app.deploymentSpec());
+            for (var routingMethod : availableRoutingMethods) {
+                endpoints.add(Endpoint.of(applicationId)
+                                      .targetApplication(EndpointId.of(declaredEndpoint.endpointId()),
+                                                         ClusterSpec.Id.from(declaredEndpoint.containerId()),
+                                                         deployments)
+                                      .routingMethod(routingMethod)
+                                      .on(Port.fromRoutingMethod(routingMethod))
+                                      .in(controller.system()));
+            }
+        }
+        return EndpointList.copyOf(endpoints);
+    }
+
     /** Returns all zone-scoped endpoints and corresponding cluster IDs for given deployments, grouped by their zone */
     public Map<ZoneId, List<Endpoint>> zoneEndpointsOf(Collection<DeploymentId> deployments) {
         var endpoints = new TreeMap<ZoneId, List<Endpoint>>(Comparator.comparing(ZoneId::value));
@@ -287,7 +310,7 @@ public class RoutingController {
     }
 
     /** Returns the routing methods that are available across all given deployments */
-    private List<RoutingMethod> routingMethodsOfAll(List<DeploymentId> deployments, DeploymentSpec deploymentSpec) {
+    private List<RoutingMethod> routingMethodsOfAll(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) {
         var deploymentsByMethod = new HashMap<RoutingMethod, Set<DeploymentId>>();
         for (var deployment : deployments) {
             for (var method : controller.zoneRegistry().routingMethods(deployment.zoneId())) {
@@ -306,7 +329,7 @@ public class RoutingController {
     }
 
     /** Returns whether traffic can be directly routed to all given deployments */
-    private boolean canRouteDirectlyTo(List<DeploymentId> deployments, DeploymentSpec deploymentSpec) {
+    private boolean canRouteDirectlyTo(Collection<DeploymentId> deployments, DeploymentSpec deploymentSpec) {
         return deployments.stream().allMatch(deployment -> canRouteDirectlyTo(deployment, deploymentSpec));
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java
index 2c59211e50c..aecb1e7a2c1 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/NameServiceForwarder.java
@@ -73,6 +73,11 @@ public class NameServiceForwarder {
         forward(new RemoveRecords(type, data), priority);
     }
 
+    /** Remove all records of given type, name and data */
+    public void removeRecords(Record.Type type, RecordName name, RecordData data, NameServiceQueue.Priority priority) {
+        forward(new RemoveRecords(type, name, data), priority);
+    }
+
     protected void forward(NameServiceRequest request, NameServiceQueue.Priority priority) {
         try (Lock lock = db.lockNameServiceQueue()) {
             NameServiceQueue queue = db.readNameServiceQueue();
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java
index 6bc67af5b98..f940d53fab3 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/dns/RemoveRecords.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.vespa.hosted.controller.dns;
 
+import com.yahoo.vespa.hosted.controller.api.integration.dns.AliasTarget;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.NameService;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
@@ -12,7 +13,11 @@ import java.util.Objects;
 import java.util.Optional;
 
 /**
- * Permanently removes all matching records by type and name or data.
+ * Permanently removes all matching records by type and matching either:
+ *
+ * - name and data
+ * - only name
+ * - only data
  *
  * @author mpolden
  */
@@ -30,13 +35,17 @@ public class RemoveRecords implements NameServiceRequest {
         this(type, Optional.empty(), Optional.of(data));
     }
 
+    public RemoveRecords(Record.Type type, RecordName name, RecordData data) {
+        this(type, Optional.of(name), Optional.of(data));
+    }
+
     /** DO NOT USE. Public for serialization purposes */
     public RemoveRecords(Record.Type type, Optional<RecordName> name, Optional<RecordData> data) {
         this.type = Objects.requireNonNull(type, "type must be non-null");
         this.name = Objects.requireNonNull(name, "name must be non-null");
         this.data = Objects.requireNonNull(data, "data must be non-null");
-        if (name.isPresent() == data.isPresent()) {
-            throw new IllegalArgumentException("exactly one of name or data must be non-empty");
+        if (name.isEmpty() && data.isEmpty()) {
+            throw new IllegalArgumentException("at least one of name and data must be non-empty");
         }
     }
 
@@ -55,8 +64,23 @@ public class RemoveRecords implements NameServiceRequest {
     @Override
     public void dispatchTo(NameService nameService) {
         List<Record> records = new ArrayList<>();
-        name.ifPresent(n -> records.addAll(nameService.findRecords(type, n)));
-        data.ifPresent(d -> records.addAll(nameService.findRecords(type, d)));
+        if (name.isPresent() && data.isPresent()) {
+            nameService.findRecords(type, name.get())
+                       .stream()
+                       .filter(record -> {
+                           // Records to remove must match both name and data fields
+                           String dataValue = record.data().asString();
+                           // If we're comparing an ALIAS record we have to unpack it to access the target name
+                           if (record.type() == Record.Type.ALIAS) {
+                               dataValue = AliasTarget.unpack(record.data()).name().value();
+                           }
+                           return fqdn(dataValue).equals(fqdn(data.get().asString()));
+                       })
+                       .forEach(records::add);
+        } else {
+            name.ifPresent(n -> records.addAll(nameService.findRecords(type, n)));
+            data.ifPresent(d -> records.addAll(nameService.findRecords(type, d)));
+        }
         nameService.removeRecords(records);
     }
 
@@ -82,4 +106,8 @@ public class RemoveRecords implements NameServiceRequest {
         return Objects.hash(type, name, data);
     }
 
+    private static String fqdn(String name) {
+        return name.endsWith(".") ? name : name + ".";
+    }
+
 }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 2845dd53b24..17b15230458 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1563,7 +1563,7 @@ public class ApplicationApiHandler extends AuditLoggingRequestHandler {
     private void setGlobalEndpointStatus(DeploymentId deployment, boolean inService, HttpRequest request) {
         var agent = isOperator(request) ? GlobalRouting.Agent.operator : GlobalRouting.Agent.tenant;
         var status = inService ? GlobalRouting.Status.in : GlobalRouting.Status.out;
-        controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent);
+        controller.routing().policies().setRoutingStatus(deployment, status, agent);
     }
 
     /** Set the global rotation status for given deployment. This only applies to global endpoints backed by a rotation */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
index 34fcda3bff8..334ab0b9f73 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/routing/RoutingApiHandler.java
@@ -211,7 +211,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         var zone = zoneFrom(path);
         if (controller.zoneRegistry().zones().directlyRouted().ids().contains(zone)) {
             var status = in ? GlobalRouting.Status.in : GlobalRouting.Status.out;
-            controller.routing().policies().setGlobalRoutingStatus(zone, status);
+            controller.routing().policies().setRoutingStatus(zone, status);
         } else {
             controller.serviceRegistry().configServer().setGlobalRotationStatus(zone, in);
         }
@@ -256,7 +256,7 @@ public class RoutingApiHandler extends AuditLoggingRequestHandler {
         }
 
         // Set policy status
-        controller.routing().policies().setGlobalRoutingStatus(deployment, status, agent);
+        controller.routing().policies().setRoutingStatus(deployment, status, agent);
         return new MessageResponse("Set global routing status for " + deployment + " to " + (in ? "IN" : "OUT"));
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
index 343fa5417ce..67eafe6235d 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingId.java
@@ -3,22 +3,30 @@ package com.yahoo.vespa.hosted.controller.routing;
 
 import com.yahoo.config.provision.ApplicationId;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 
 import java.util.Objects;
 
 /**
- * Unique identifier for a global routing table entry (instance x endpoint ID).
+ * Unique identifier for a instance routing table entry (instance x endpoint ID).
  *
  * @author mpolden
  */
 public class RoutingId {
 
+    private final TenantAndApplicationId application;
     private final ApplicationId instance;
     private final EndpointId endpointId;
 
-    public RoutingId(ApplicationId instance, EndpointId endpointId) {
-        this.instance = Objects.requireNonNull(instance, "instance must be non-null");
+    private RoutingId(ApplicationId instance, EndpointId endpointId) {
+        this.instance = Objects.requireNonNull(instance, "application must be non-null");
         this.endpointId = Objects.requireNonNull(endpointId, "endpointId must be non-null");
+
+        application = TenantAndApplicationId.from(instance);
+    }
+
+    public TenantAndApplicationId application() {
+        return application;
     }
 
     public ApplicationId instance() {
@@ -33,14 +41,13 @@ public class RoutingId {
     public boolean equals(Object o) {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
-        RoutingId that = (RoutingId) o;
-        return instance.equals(that.instance) &&
-               endpointId.equals(that.endpointId);
+        RoutingId routingId = (RoutingId) o;
+        return application.equals(routingId.application) && instance.equals(routingId.instance) && endpointId.equals(routingId.endpointId);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(instance, endpointId);
+        return Objects.hash(application, instance, endpointId);
     }
 
     @Override
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
index e0c0df5234e..fdfb224b8ca 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicies.java
@@ -18,6 +18,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.WeightedAliasTarget;
 import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
+import com.yahoo.vespa.hosted.controller.application.EndpointList;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceForwarder;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceQueue.Priority;
 import com.yahoo.vespa.hosted.controller.dns.NameServiceRequest;
@@ -32,11 +34,12 @@ import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
 
 /**
- * Updates routing policies and their associated DNS records based on an deployment's load balancers.
+ * Updates routing policies and their associated DNS records based on a deployment's load balancers.
  *
  * @author mortent
  * @author mpolden
@@ -75,24 +78,29 @@ public class RoutingPolicies {
     }
 
     /**
-     * Refresh routing policies for application in given zone. This is idempotent and changes will only be performed if
-     * load balancers for given application have changed.
+     * Refresh routing policies for instance in given zone. This is idempotent and changes will only be performed if
+     * load balancers for given instance have changed.
      */
-    public void refresh(ApplicationId application, DeploymentSpec deploymentSpec, ZoneId zone) {
-        var allocation = new LoadBalancerAllocation(application, zone, controller.serviceRegistry().configServer()
-                                                                                 .getLoadBalancers(application, zone),
-                                                    deploymentSpec);
-        var inactiveZones = inactiveZones(application, deploymentSpec);
+    public void refresh(ApplicationId instance, DeploymentSpec deploymentSpec, ZoneId zone) {
+        LoadBalancerAllocation allocation = new LoadBalancerAllocation(instance, zone, controller.serviceRegistry().configServer()
+                                                                                                    .getLoadBalancers(instance, zone),
+                                                                       deploymentSpec);
+        Set<ZoneId> inactiveZones = inactiveZones(instance, deploymentSpec);
         try (var lock = db.lockRoutingPolicies()) {
             removeGlobalDnsUnreferencedBy(allocation, lock);
+            removeApplicationDnsUnreferencedBy(allocation, lock);
+
             storePoliciesOf(allocation, lock);
             removePoliciesUnreferencedBy(allocation, lock);
-            updateGlobalDnsOf(get(allocation.deployment.applicationId()).values(), inactiveZones, lock);
+
+            Collection<RoutingPolicy> policies = get(allocation.deployment.applicationId()).values();
+            updateGlobalDnsOf(policies, inactiveZones, lock);
+            updateApplicationDnsOf(policies, inactiveZones, lock);
         }
     }
 
     /** Set the status of all global endpoints in given zone */
-    public void setGlobalRoutingStatus(ZoneId zone, GlobalRouting.Status status) {
+    public void setRoutingStatus(ZoneId zone, GlobalRouting.Status status) {
         try (var lock = db.lockRoutingPolicies()) {
             db.writeZoneRoutingPolicy(new ZoneRoutingPolicy(zone, GlobalRouting.status(status, GlobalRouting.Agent.operator,
                                                                                        controller.clock().instant())));
@@ -104,24 +112,25 @@ public class RoutingPolicies {
     }
 
     /** Set the status of all global endpoints for given deployment */
-    public void setGlobalRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) {
+    public void setRoutingStatus(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) {
         try (var lock = db.lockRoutingPolicies()) {
             var policies = get(deployment.applicationId());
             var newPolicies = new LinkedHashMap<>(policies);
             for (var policy : policies.values()) {
-                if (!policy.id().zone().equals(deployment.zoneId())) continue; // Wrong zone
+                if (!policy.appliesTo(deployment)) continue;
                 var newPolicy = policy.with(policy.status().with(GlobalRouting.status(status, agent,
                                                                                       controller.clock().instant())));
                 newPolicies.put(policy.id(), newPolicy);
             }
             db.writeRoutingPolicies(deployment.applicationId(), newPolicies);
             updateGlobalDnsOf(newPolicies.values(), Set.of(), lock);
+            updateApplicationDnsOf(newPolicies.values(), Set.of(), lock);
         }
     }
 
     /** Update global DNS records for given policies */
     private void updateGlobalDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
-        Map<RoutingId, List<RoutingPolicy>> routingTable = routingTableFrom(routingPolicies);
+        Map<RoutingId, List<RoutingPolicy>> routingTable = instanceRoutingTable(routingPolicies);
         for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
             RoutingId routingId = routeEntry.getKey();
             controller.routing().endpointsOf(routingId.instance())
@@ -167,7 +176,6 @@ public class RoutingPolicies {
                                                                      Priority.normal));
     }
 
-
     /** Compute region endpoints and their targets from given policies */
     private Collection<RegionEndpoint> computeRegionEndpoints(List<RoutingPolicy> policies, Set<ZoneId> inactiveZones) {
         Map<Endpoint, RegionEndpoint> endpoints = new LinkedHashMap<>();
@@ -178,8 +186,8 @@ public class RoutingPolicies {
             Endpoint regionEndpoint = policy.regionEndpointIn(controller.system(), routingMethod);
             var zonePolicy = db.readZoneRoutingPolicy(policy.id().zone());
             long weight = 1;
-            if (isConfiguredOut(policy, zonePolicy, inactiveZones)) {
-                weight = 0; // A record with 0 weight will not received traffic. If all records within a group have 0
+            if (isConfiguredOut(zonePolicy, policy, inactiveZones)) {
+                weight = 0; // A record with 0 weight will not receive traffic. If all records within a group have 0
                             // weight, traffic is routed to all records with equal probability.
             }
             var weightedTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(),
@@ -193,6 +201,43 @@ public class RoutingPolicies {
         return endpoints.values();
     }
 
+
+    private void updateApplicationDnsOf(Collection<RoutingPolicy> routingPolicies, Set<ZoneId> inactiveZones, @SuppressWarnings("unused") Lock lock) {
+        // In the context of single deployment (which this is) there is only one routing policy per routing ID. I.e.
+        // there is no scenario where more than one deployment within an instance can be a member the same
+        // application-level endpoint. However, to allow this in the future the routing table remains
+        // Map<RoutingId, List<RoutingPolicy>> instead of Map<RoutingId, RoutingPolicy>.
+        Map<RoutingId, List<RoutingPolicy>> routingTable = applicationRoutingTable(routingPolicies);
+        Map<String, Set<AliasTarget>> targetsByEndpoint = new LinkedHashMap<>();
+        for (Map.Entry<RoutingId, List<RoutingPolicy>> routeEntry : routingTable.entrySet()) {
+            RoutingId routingId = routeEntry.getKey();
+            EndpointList endpoints = controller.routing().endpointsOf(routingId.application())
+                                               .named(routingId.endpointId());
+            if (endpoints.isEmpty()) continue;
+            if (endpoints.size() > 1) {
+                throw new IllegalArgumentException("Expected at most 1 endpoint with ID '" + routingId.endpointId() +
+                                                   ", got " + endpoints.size());
+            }
+            Endpoint endpoint = endpoints.asList().get(0);
+            for (var policy : routeEntry.getValue()) {
+                for (var target : endpoint.targets()) {
+                    if (!policy.appliesTo(target.deployment())) continue;
+                    int weight = target.weight();
+                    if (isConfiguredOut(policy, inactiveZones)) {
+                        weight = 0;
+                    }
+                    WeightedAliasTarget weightedAliasTarget = new WeightedAliasTarget(policy.canonicalName(), policy.dnsZone().get(),
+                                                                                      target.deployment().zoneId(), weight);
+                    targetsByEndpoint.computeIfAbsent(endpoint.dnsName(), (k) -> new LinkedHashSet<>())
+                                     .add(weightedAliasTarget);
+                }
+            }
+        }
+        targetsByEndpoint.forEach((applicationEndpoint, targets) -> {
+            controller.nameServiceForwarder().createAlias(RecordName.from(applicationEndpoint), targets, Priority.normal);
+        });
+    }
+
     /** Store routing policies for given load balancers */
     private void storePoliciesOf(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) {
         var policies = new LinkedHashMap<>(get(allocation.deployment.applicationId()));
@@ -201,8 +246,8 @@ public class RoutingPolicies {
             var policyId = new RoutingPolicyId(loadBalancer.application(), loadBalancer.cluster(), allocation.deployment.zoneId());
             var existingPolicy = policies.get(policyId);
             var newPolicy = new RoutingPolicy(policyId, loadBalancer.hostname().get(), loadBalancer.dnsZone(),
-                                              allocation.endpointIdsOf(loadBalancer),
-                                              Set.of(),
+                                              allocation.instanceEndpointsOf(loadBalancer),
+                                              allocation.applicationEndpointsOf(loadBalancer),
                                               new Status(isActive(loadBalancer), GlobalRouting.DEFAULT_STATUS));
             // Preserve global routing status for existing policy
             if (existingPolicy != null) {
@@ -230,8 +275,7 @@ public class RoutingPolicies {
         var activeIds = allocation.asPolicyIds();
         for (var policy : policies.values()) {
             // Leave active load balancers and irrelevant zones alone
-            if (activeIds.contains(policy.id()) ||
-                !policy.id().zone().equals(allocation.deployment.zoneId())) continue;
+            if (activeIds.contains(policy.id()) || !policy.appliesTo(allocation.deployment)) continue;
             for (var endpoint : policy.zoneEndpointsIn(controller.system(), RoutingMethod.exclusive, controller.zoneRegistry())) {
                 var dnsName = endpoint.dnsName();
                 nameServiceForwarderIn(allocation.deployment.zoneId()).removeRecords(Record.Type.CNAME,
@@ -243,39 +287,86 @@ public class RoutingPolicies {
         db.writeRoutingPolicies(allocation.deployment.applicationId(), newPolicies);
     }
 
-    /** Remove unreferenced global endpoints from DNS */
+    /** Remove unreferenced instance endpoints from DNS */
     private void removeGlobalDnsUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) {
-        var zonePolicies = get(allocation.deployment).values();
-        var removalCandidates = new HashSet<>(routingTableFrom(zonePolicies).keySet());
-        var activeRoutingIds = routingIdsFrom(allocation);
+        Collection<RoutingPolicy> zonePolicies = get(allocation.deployment).values();
+        Set<RoutingId> removalCandidates = new HashSet<>(instanceRoutingTable(zonePolicies).keySet());
+        Set<RoutingId> activeRoutingIds = instanceRoutingIds(allocation);
         removalCandidates.removeAll(activeRoutingIds);
         for (var id : removalCandidates) {
-            var endpoints = controller.routing().endpointsOf(id.instance())
-                                      .not().requiresRotation()
-                                      .named(id.endpointId());
-            var forwarder = nameServiceForwarderIn(allocation.deployment.zoneId());
+            EndpointList endpoints = controller.routing().endpointsOf(id.instance())
+                                               .not().requiresRotation()
+                                               .named(id.endpointId());
+            NameServiceForwarder forwarder = nameServiceForwarderIn(allocation.deployment.zoneId());
+            // This removes all ALIAS records having this DNS name. There is no attempt to delete only the entry for the
+            // affected zone. Instead, the correct set of records is (re)created by updateGlobalDnsOf
             endpoints.forEach(endpoint -> forwarder.removeRecords(Record.Type.ALIAS, RecordName.from(endpoint.dnsName()),
                                                                   Priority.normal));
         }
     }
 
+    /** Remove unreferenced application endpoints in given allocation from DNS */
+    private void removeApplicationDnsUnreferencedBy(LoadBalancerAllocation allocation, @SuppressWarnings("unused") Lock lock) {
+        Collection<RoutingPolicy> zonePolicies = get(allocation.deployment).values();
+        Map<RoutingId, List<RoutingPolicy>> routingTable = applicationRoutingTable(zonePolicies);
+        Set<RoutingId> removalCandidates = new HashSet<>(routingTable.keySet());
+        Set<RoutingId> activeRoutingIds = applicationRoutingIds(allocation);
+        removalCandidates.removeAll(activeRoutingIds);
+        for (var id : removalCandidates) {
+            TenantAndApplicationId application = TenantAndApplicationId.from(id.instance());
+            EndpointList endpoints = controller.routing()
+                                               .endpointsOf(application)
+                                               .named(id.endpointId());
+            List<RoutingPolicy> policies = routingTable.get(id);
+            for (var policy : policies) {
+                if (!policy.appliesTo(allocation.deployment)) continue;
+                NameServiceForwarder forwarder = nameServiceForwarderIn(policy.id().zone());
+                endpoints.forEach(endpoint -> forwarder.removeRecords(Record.Type.ALIAS,
+                                                                      RecordName.from(endpoint.dnsName()),
+                                                                      RecordData.fqdn(policy.canonicalName().value()),
+                                                                      Priority.normal));
+            }
+        }
+    }
+
+    private Set<RoutingId> instanceRoutingIds(LoadBalancerAllocation allocation) {
+        return routingIdsFrom(allocation, false);
+    }
+
+    private Set<RoutingId> applicationRoutingIds(LoadBalancerAllocation allocation) {
+        return routingIdsFrom(allocation, true);
+    }
+
     /** Compute routing IDs from given load balancers */
-    private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation) {
+    private static Set<RoutingId> routingIdsFrom(LoadBalancerAllocation allocation, boolean applicationLevel) {
         Set<RoutingId> routingIds = new LinkedHashSet<>();
         for (var loadBalancer : allocation.loadBalancers) {
-            for (var endpointId : allocation.endpointIdsOf(loadBalancer)) {
-                routingIds.add(new RoutingId(loadBalancer.application(), endpointId));
+            Set<EndpointId> endpoints = applicationLevel
+                    ? allocation.applicationEndpointsOf(loadBalancer)
+                    : allocation.instanceEndpointsOf(loadBalancer);
+            for (var endpointId : endpoints) {
+                routingIds.add(RoutingId.of(loadBalancer.application(), endpointId));
             }
         }
         return Collections.unmodifiableSet(routingIds);
     }
 
-    /** Compute a routing table from given policies */
-    private static Map<RoutingId, List<RoutingPolicy>> routingTableFrom(Collection<RoutingPolicy> routingPolicies) {
-        var routingTable = new LinkedHashMap<RoutingId, List<RoutingPolicy>>();
+    /** Compute a routing table for instance-level endpoints from given policies */
+    private static Map<RoutingId, List<RoutingPolicy>> instanceRoutingTable(Collection<RoutingPolicy> routingPolicies) {
+        return routingTable(routingPolicies, false);
+    }
+
+    /** Compute a routing table for application-level endpoints from given policies */
+    private static Map<RoutingId, List<RoutingPolicy>> applicationRoutingTable(Collection<RoutingPolicy> routingPolicies) {
+        return routingTable(routingPolicies, true);
+    }
+
+    private static Map<RoutingId, List<RoutingPolicy>> routingTable(Collection<RoutingPolicy> routingPolicies, boolean applicationLevel) {
+        Map<RoutingId, List<RoutingPolicy>> routingTable = new LinkedHashMap<>();
         for (var policy : routingPolicies) {
-            for (var endpoint : policy.instanceEndpoints()) {
-                var id = new RoutingId(policy.id().owner(), endpoint);
+            Set<EndpointId> endpoints = applicationLevel ? policy.applicationEndpoints() : policy.instanceEndpoints();
+            for (var endpoint : endpoints) {
+                RoutingId id = RoutingId.of(policy.id().owner(), endpoint);
                 routingTable.computeIfAbsent(id, k -> new ArrayList<>())
                             .add(policy);
             }
@@ -283,13 +374,22 @@ public class RoutingPolicies {
         return Collections.unmodifiableMap(routingTable);
     }
 
-    /** Returns whether the global routing status of given policy is configured to be {@link GlobalRouting.Status#out} */
-    private static boolean isConfiguredOut(RoutingPolicy policy, ZoneRoutingPolicy zonePolicy, Set<ZoneId> inactiveZones) {
-        // A deployment is can be configured out at any of the following levels:
-        // - zone level (ZoneRoutingPolicy)
+    /** Returns whether the endpoints of given policy are globally configured {@link GlobalRouting.Status#out} */
+    private static boolean isConfiguredOut(ZoneRoutingPolicy zonePolicy, RoutingPolicy policy, Set<ZoneId> inactiveZones) {
+        return isConfiguredOut(policy, Optional.of(zonePolicy), inactiveZones);
+    }
+
+    /** Returns whether the endpoints of given policy are configured {@link GlobalRouting.Status#out} */
+    private static boolean isConfiguredOut(RoutingPolicy policy, Set<ZoneId> inactiveZones) {
+        return isConfiguredOut(policy, Optional.empty(), inactiveZones);
+    }
+
+    private static boolean isConfiguredOut(RoutingPolicy policy, Optional<ZoneRoutingPolicy> zonePolicy, Set<ZoneId> inactiveZones) {
+        // A deployment can be configured out from endpoints at any of the following levels:
+        // - zone level (ZoneRoutingPolicy, only applies to global endpoints)
         // - deployment level (RoutingPolicy)
         // - application package level (deployment.xml)
-        return zonePolicy.globalRouting().status() == GlobalRouting.Status.out ||
+        return (zonePolicy.isPresent() && zonePolicy.get().globalRouting().status() == GlobalRouting.Status.out) ||
                policy.status().globalRouting().status() == GlobalRouting.Status.out ||
                inactiveZones.contains(policy.id().zone());
     }
@@ -363,8 +463,8 @@ public class RoutingPolicies {
                                 .collect(Collectors.toUnmodifiableSet());
         }
 
-        /** Compute all endpoint IDs for given load balancer */
-        private Set<EndpointId> endpointIdsOf(LoadBalancer loadBalancer) {
+        /** Returns all instance endpoint IDs served by given load balancer */
+        private Set<EndpointId> instanceEndpointsOf(LoadBalancer loadBalancer) {
             if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints
                 return Set.of();
             }
@@ -384,6 +484,21 @@ public class RoutingPolicies {
                                .collect(Collectors.toUnmodifiableSet());
         }
 
+        /** Returns all application endpoint IDs served by given load balancer */
+        private Set<EndpointId> applicationEndpointsOf(LoadBalancer loadBalancer) {
+            if (!deployment.zoneId().environment().isProduction()) { // Only production deployments have configurable endpoints
+                return Set.of();
+            }
+            return deploymentSpec.endpoints().stream()
+                                 .filter(endpoint -> endpoint.containerId().equals(loadBalancer.cluster().value()))
+                                 .filter(endpoint -> endpoint.targets().stream()
+                                                             .anyMatch(target -> target.region().equals(deployment.zoneId().region()) &&
+                                                                                 target.instance().equals(deployment.applicationId().instance())))
+                                 .map(com.yahoo.config.application.api.Endpoint::endpointId)
+                                 .map(EndpointId::of)
+                                 .collect(Collectors.toUnmodifiableSet());
+        }
+
     }
 
     /** Returns zones where global routing is declared inactive for instance through deploymentSpec */
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
index 5653b51f6c9..ba8bf9c88ba 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/routing/RoutingPolicy.java
@@ -75,6 +75,12 @@ public class RoutingPolicy {
         return status;
     }
 
+    /** Returns whether this policy applies to given deployment */
+    public boolean appliesTo(DeploymentId deployment) {
+        return id.owner().equals(deployment.applicationId()) &&
+               id.zone().equals(deployment.zoneId());
+    }
+
     /** Returns a copy of this with status set to given status */
     public RoutingPolicy with(Status status) {
         return new RoutingPolicy(id, canonicalName, dnsZone, instanceEndpoints, applicationEndpoints, status);
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index 64821756105..91a12d3b465 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.security.SignatureAlgorithm;
 import com.yahoo.security.X509CertificateBuilder;
@@ -26,8 +27,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.OptionalInt;
 import java.util.StringJoiner;
+import java.util.TreeMap;
 import java.util.zip.Deflater;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -48,6 +51,7 @@ public class ApplicationPackageBuilder {
                                                                 "<notifications>\n  <email ",
                                                                 "/>\n</notifications>\n").setEmptyValue("");
     private final StringBuilder endpointsBody = new StringBuilder();
+    private final StringBuilder applicationEndpointsBody = new StringBuilder();
     private final List<X509Certificate> trustedCertificates = new ArrayList<>();
 
     private OptionalInt majorVersion = OptionalInt.empty();
@@ -86,9 +90,9 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
-    public ApplicationPackageBuilder endpoint(String endpointId, String containerId, String... regions) {
+    public ApplicationPackageBuilder endpoint(String id, String containerId, String... regions) {
         endpointsBody.append("      <endpoint");
-        endpointsBody.append(" id='").append(endpointId).append("'");
+        endpointsBody.append(" id='").append(id).append("'");
         endpointsBody.append(" container-id='").append(containerId).append("'");
         endpointsBody.append(">\n");
         for (var region : regions) {
@@ -98,6 +102,23 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
+    public ApplicationPackageBuilder applicationEndpoint(String id, String containerId, String region,
+                                                         Map<InstanceName, Integer> instanceWeights) {
+        if (instanceWeights.isEmpty()) throw new IllegalArgumentException("At least one instance must be given");
+        applicationEndpointsBody.append("    <endpoint");
+        applicationEndpointsBody.append(" id='").append(id).append("'");
+        applicationEndpointsBody.append(" container-id='").append(containerId).append("'");
+        applicationEndpointsBody.append(" region='").append(region).append("'");
+        applicationEndpointsBody.append(">\n");
+        for (var kv : new TreeMap<>(instanceWeights).entrySet()) {
+            applicationEndpointsBody.append("      <instance weight='").append(kv.getValue().toString()).append("'>")
+                                    .append(kv.getKey().value())
+                                    .append("</instance>\n");
+        }
+        applicationEndpointsBody.append("    </endpoint>\n");
+        return this;
+    }
+
     public ApplicationPackageBuilder systemTest() {
         explicitSystemTest = true;
         return this;
@@ -248,10 +269,17 @@ public class ApplicationPackageBuilder {
         xml.append(">\n");
         xml.append(prodBody);
         xml.append("    </prod>\n");
-        xml.append("    <endpoints>\n");
-        xml.append(endpointsBody);
-        xml.append("    </endpoints>\n");
+        if (endpointsBody.length() > 0 ) {
+            xml.append("    <endpoints>\n");
+            xml.append(endpointsBody);
+            xml.append("    </endpoints>\n");
+        }
         xml.append("  </instance>\n");
+        if (applicationEndpointsBody.length() > 0) {
+            xml.append("  <endpoints>\n");
+            xml.append(applicationEndpointsBody);
+            xml.append("  </endpoints>\n");
+        }
         xml.append("</deployment>\n");
         return xml.toString().getBytes(UTF_8);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index e2ef27492af..5fffa640811 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
@@ -165,8 +166,8 @@ public class RoutingPoliciesTest {
                      tester.policiesOf(context.instance().id()).size());
 
         // A zone in shared region is set out
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.out,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
 
         // Weight of inactive zone is set to zero
@@ -176,16 +177,16 @@ public class RoutingPoliciesTest {
 
         // Other zone in shared region is set out. Entire record group for the region is removed as all zones in the
         // region are out (weight sum = 0)
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.out,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L));
 
         // Everything is set back in
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.in,
+                                                  GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.in,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L,
                                                                                            zone3, 1L,
@@ -480,8 +481,8 @@ public class RoutingPoliciesTest {
 
         // Global routing status is overridden in one zone
         var changedAt = tester.controllerTester().clock().instant();
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
 
         // Inactive zone is removed from global DNS record
@@ -509,7 +510,7 @@ public class RoutingPoliciesTest {
         // Deployment is set back in
         tester.controllerTester().clock().advance(Duration.ofHours(1));
         changedAt = tester.controllerTester().clock().instant();
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
@@ -562,7 +563,7 @@ public class RoutingPoliciesTest {
         }
 
         // Set zone out
-        tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out);
+        tester.routingPolicies().setRoutingStatus(zone2, GlobalRouting.Status.out);
         context1.flushDnsUpdates();
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
@@ -582,17 +583,17 @@ public class RoutingPoliciesTest {
         assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt());
 
         // Setting status per deployment does not affect status as entire zone is out
-        tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
         context1.flushDnsUpdates();
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
 
         // Set single deployment out
-        tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
         context1.flushDnsUpdates();
 
         // Set zone back in. Deployment set explicitly out, remains out, the rest are in
-        tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in);
+        tester.routingPolicies().setRoutingStatus(zone2, GlobalRouting.Status.in);
         context1.flushDnsUpdates();
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
@@ -646,38 +647,38 @@ public class RoutingPoliciesTest {
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
 
         // Global routing status is overridden for one deployment
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
 
         // Setting other deployment out implicitly sets all deployments in. Weight is set to zero, but that has no
         // impact on routing decisions when the weight sum is zero
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L));
 
         // One inactive deployment is put back in. Global DNS record now points to the only active deployment
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
 
         // Setting zone (containing active deployment) out puts all deployments in
-        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out);
+        tester.routingPolicies().setRoutingStatus(zone1, GlobalRouting.Status.out);
         context.flushDnsUpdates();
         assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status());
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L));
 
         // Setting zone back in removes the currently inactive deployment
-        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in);
+        tester.routingPolicies().setRoutingStatus(zone1, GlobalRouting.Status.in);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
 
         // Inactive deployment is set in
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in,
+                                                  GlobalRouting.Agent.tenant);
         context.flushDnsUpdates();
         for (var policy : tester.routingPolicies().get(context.instanceId()).values()) {
             assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status());
@@ -701,12 +702,103 @@ public class RoutingPoliciesTest {
                      records.get(0).data());
     }
 
+    @Test
+    public void application_endpoint_routing_policy() {
+        RoutingPoliciesTester tester = new RoutingPoliciesTester();
+        TenantAndApplicationId application = TenantAndApplicationId.from("tenant1", "app1");
+        ApplicationId betaInstance = application.instance("beta");
+        ApplicationId mainInstance = application.instance("main");
+
+        DeploymentContext betaContext = tester.newDeploymentContext(betaInstance);
+        DeploymentContext mainContext = tester.newDeploymentContext(mainInstance);
+        var applicationPackage = applicationPackageBuilder()
+                .instances("beta,main")
+                .region(zone1.region())
+                .region(zone2.region())
+                .applicationEndpoint("a0", "c0", "us-west-1",
+                                     Map.of(betaInstance.instance(), 2,
+                                            mainInstance.instance(), 8))
+                .applicationEndpoint("a1", "c1", "us-central-1",
+                                     Map.of(betaInstance.instance(), 4,
+                                            mainInstance.instance(), 6))
+                .build();
+        for (var zone : List.of(zone1, zone2)) {
+            tester.provisionLoadBalancers(2, betaInstance, zone);
+            tester.provisionLoadBalancers(2, mainInstance, zone);
+        }
+
+        // Deploy both instances
+        betaContext.submit(applicationPackage).deploy();
+
+        // Application endpoint points to both instances with correct weights
+        DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1);
+        DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1);
+        DeploymentId betaZone2 = betaContext.deploymentIdIn(zone2);
+        DeploymentId mainZone2 = mainContext.deploymentIdIn(zone2);
+        tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
+                             Map.of(betaZone1, 2,
+                                    mainZone1, 8));
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 4,
+                                    mainZone2, 6));
+
+        // Weights are updated
+        applicationPackage = applicationPackageBuilder()
+                .instances("beta,main")
+                .region(zone1.region())
+                .region(zone2.region())
+                .applicationEndpoint("a0", "c0", "us-west-1",
+                                     Map.of(betaInstance.instance(), 3,
+                                            mainInstance.instance(), 7))
+                .applicationEndpoint("a1", "c1", "us-central-1",
+                                     Map.of(betaInstance.instance(), 1,
+                                            mainInstance.instance(), 9))
+                .build();
+        betaContext.submit(applicationPackage).deploy();
+        tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
+                             Map.of(betaZone1, 3,
+                                    mainZone1, 7));
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 1,
+                                    mainZone2, 9));
+
+        // Changing routing status updates weight
+        tester.routingPolicies().setRoutingStatus(mainZone2, GlobalRouting.Status.out, RoutingStatus.Agent.tenant);
+        betaContext.flushDnsUpdates();
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 1,
+                                    mainZone2, 0));
+        tester.routingPolicies().setRoutingStatus(mainZone2, GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        betaContext.flushDnsUpdates();
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 1,
+                                    mainZone2, 9));
+
+        // An endpoint is removed
+        applicationPackage = applicationPackageBuilder()
+                .instances("beta,main")
+                .region(zone1.region())
+                .region(zone2.region())
+                .applicationEndpoint("a0", "c0", "us-west-1",
+                                     Map.of(betaInstance.instance(), 1))
+                .build();
+        betaContext.submit(applicationPackage).deploy();
+
+        // Application endpoints now point to a single instance
+        tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
+                             Map.of(betaZone1, 1));
+        assertTrue("Endpoint removed",
+                   tester.controllerTester().controller().routing()
+                         .endpointsOf(application)
+                         .named(EndpointId.of("a1")).isEmpty());
+    }
+
     /** Returns an application package builder that satisfies requirements for a directly routed endpoint */
     private static ApplicationPackageBuilder applicationPackageBuilder() {
-        return new ApplicationPackageBuilder()
-                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"));
+        return new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"),
+                                                              AthenzService.from("service"));
     }
-    
+
     private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) {
         List<LoadBalancer> loadBalancers = new ArrayList<>();
         for (int i = 0; i < count; i++) {
@@ -819,7 +911,35 @@ public class RoutingPoliciesTest {
                          .collect(Collectors.toList());
         }
 
-        private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map<ZoneId, Long> zoneWeights) {
+        /** Assert that an application endpoint points to given targets and weights */
+        private void assertTargets(TenantAndApplicationId application, EndpointId endpointId, ClusterSpec.Id cluster,
+                                   int loadBalancerId, Map<DeploymentId, Integer> deploymentWeights) {
+            Map<String, List<DeploymentId>> deploymentsByDnsName = new HashMap<>();
+            for (var deployment : deploymentWeights.keySet()) {
+                EndpointList applicationEndpoints = tester.controller().routing().endpointsOf(application)
+                                                          .named(endpointId)
+                                                          .targets(deployment)
+                                                          .cluster(cluster);
+                assertEquals("Expected a single endpoint with ID '" + endpointId + "'", 1,
+                             applicationEndpoints.size());
+                String dnsName = applicationEndpoints.asList().get(0).dnsName();
+                deploymentsByDnsName.computeIfAbsent(dnsName, (k) -> new ArrayList<>())
+                                    .add(deployment);
+            }
+            assertEquals("Found " + endpointId + " for " + application, 1, deploymentsByDnsName.size());
+            deploymentsByDnsName.forEach((dnsName, deployments) -> {
+                Set<String> weightedTargets = deployments.stream()
+                                                           .map(d -> "weighted/lb-" + loadBalancerId + "--" +
+                                                                     d.applicationId().serializedForm() + "--" + d.zoneId().value() +
+                                                                     "/dns-zone-1/" + d.zoneId().value() + "/" + deploymentWeights.get(d))
+                                                           .collect(Collectors.toSet());
+                assertEquals(dnsName + " has expected targets", weightedTargets, aliasDataOf(dnsName));
+            });
+        }
+
+        /** Assert that an instance endpoint points to given targets and weights */
+        private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster,
+                                   int loadBalancerId, Map<ZoneId, Long> zoneWeights) {
             Set<String> latencyTargets = new HashSet<>();
             Map<String, List<ZoneId>> zonesByRegionEndpoint = new HashMap<>();
             for (var zone : zoneWeights.keySet()) {