Merge pull request #19904 from vespa-engine/mpolden/application-level-endpoints-dns

Maintain ALIAS records for application-level endpoints

7 files changed, 254 insertions, 91 deletions

diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
index 64821756105..91a12d3b465 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/ApplicationPackageBuilder.java
@@ -5,6 +5,7 @@ import com.yahoo.component.Version;
 import com.yahoo.config.application.api.ValidationId;
 import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 import com.yahoo.security.SignatureAlgorithm;
 import com.yahoo.security.X509CertificateBuilder;
@@ -26,8 +27,10 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Date;
 import java.util.List;
+import java.util.Map;
 import java.util.OptionalInt;
 import java.util.StringJoiner;
+import java.util.TreeMap;
 import java.util.zip.Deflater;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -48,6 +51,7 @@ public class ApplicationPackageBuilder {
                                                                 "<notifications>\n  <email ",
                                                                 "/>\n</notifications>\n").setEmptyValue("");
     private final StringBuilder endpointsBody = new StringBuilder();
+    private final StringBuilder applicationEndpointsBody = new StringBuilder();
     private final List<X509Certificate> trustedCertificates = new ArrayList<>();
 
     private OptionalInt majorVersion = OptionalInt.empty();
@@ -86,9 +90,9 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
-    public ApplicationPackageBuilder endpoint(String endpointId, String containerId, String... regions) {
+    public ApplicationPackageBuilder endpoint(String id, String containerId, String... regions) {
         endpointsBody.append("      <endpoint");
-        endpointsBody.append(" id='").append(endpointId).append("'");
+        endpointsBody.append(" id='").append(id).append("'");
         endpointsBody.append(" container-id='").append(containerId).append("'");
         endpointsBody.append(">\n");
         for (var region : regions) {
@@ -98,6 +102,23 @@ public class ApplicationPackageBuilder {
         return this;
     }
 
+    public ApplicationPackageBuilder applicationEndpoint(String id, String containerId, String region,
+                                                         Map<InstanceName, Integer> instanceWeights) {
+        if (instanceWeights.isEmpty()) throw new IllegalArgumentException("At least one instance must be given");
+        applicationEndpointsBody.append("    <endpoint");
+        applicationEndpointsBody.append(" id='").append(id).append("'");
+        applicationEndpointsBody.append(" container-id='").append(containerId).append("'");
+        applicationEndpointsBody.append(" region='").append(region).append("'");
+        applicationEndpointsBody.append(">\n");
+        for (var kv : new TreeMap<>(instanceWeights).entrySet()) {
+            applicationEndpointsBody.append("      <instance weight='").append(kv.getValue().toString()).append("'>")
+                                    .append(kv.getKey().value())
+                                    .append("</instance>\n");
+        }
+        applicationEndpointsBody.append("    </endpoint>\n");
+        return this;
+    }
+
     public ApplicationPackageBuilder systemTest() {
         explicitSystemTest = true;
         return this;
@@ -248,10 +269,17 @@ public class ApplicationPackageBuilder {
         xml.append(">\n");
         xml.append(prodBody);
         xml.append("    </prod>\n");
-        xml.append("    <endpoints>\n");
-        xml.append(endpointsBody);
-        xml.append("    </endpoints>\n");
+        if (endpointsBody.length() > 0 ) {
+            xml.append("    <endpoints>\n");
+            xml.append(endpointsBody);
+            xml.append("    </endpoints>\n");
+        }
         xml.append("  </instance>\n");
+        if (applicationEndpointsBody.length() > 0) {
+            xml.append("  <endpoints>\n");
+            xml.append(applicationEndpointsBody);
+            xml.append("  </endpoints>\n");
+        }
         xml.append("</deployment>\n");
         return xml.toString().getBytes(UTF_8);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
index ba6ab4b9152..b9c1e9f3e72 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentContext.java
@@ -35,10 +35,9 @@ import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.integration.ConfigServerMock;
 import com.yahoo.vespa.hosted.controller.maintenance.JobRunner;
 import com.yahoo.vespa.hosted.controller.maintenance.NameServiceDispatcher;
-import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
-import com.yahoo.vespa.hosted.controller.routing.Status;
 
 import javax.security.auth.x500.X500Principal;
 import java.math.BigInteger;
@@ -46,6 +45,7 @@ import java.security.KeyPair;
 import java.security.cert.X509Certificate;
 import java.time.Duration;
 import java.time.Instant;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -165,13 +165,16 @@ public class DeploymentContext {
 
     /** Completely deploy the latest change */
     public DeploymentContext deploy() {
-        assertTrue("Application package submitted", application().latestVersion().isPresent());
-        assertFalse("Submission is not already deployed", application().instances().values().stream()
-                                                                       .anyMatch(instance -> instance.deployments().values().stream()
+        Application application = application();
+        assertTrue("Application package submitted", application.latestVersion().isPresent());
+        assertFalse("Submission is not already deployed", application.instances().values().stream()
+                                                                     .anyMatch(instance -> instance.deployments().values().stream()
                                                                                                      .anyMatch(deployment -> deployment.applicationVersion().equals(lastSubmission))));
-        assertEquals(application().latestVersion(), instance().change().application());
-        completeRollout();
-        assertFalse(instance().change().hasTargets());
+        assertEquals(application.latestVersion(), instance().change().application());
+        completeRollout(application.deploymentSpec().instances().size() > 1);
+        for (var instance : application().instances().values()) {
+            assertFalse(instance.change().hasTargets());
+        }
         return this;
     }
 
@@ -241,7 +244,7 @@ public class DeploymentContext {
                                            Optional.empty(),
                                            Set.of(EndpointId.of("default")),
                                            Set.of(),
-                                           new Status(false, GlobalRouting.DEFAULT_STATUS)));
+                                           new RoutingPolicy.Status(false, RoutingStatus.DEFAULT)));
         tester.controller().curator().writeRoutingPolicies(instanceId, policies);
         return this;
     }
@@ -304,19 +307,28 @@ public class DeploymentContext {
         return Optional.ofNullable(lastSubmission);
     }
 
-    /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */
     public DeploymentContext completeRollout() {
+        return completeRollout(false);
+    }
+
+    /** Runs and returns all remaining jobs for the application, at most once, and asserts the current change is rolled out. */
+    public DeploymentContext completeRollout(boolean multiInstance) {
         triggerJobs();
-        Set<JobType> jobs = new HashSet<>();
+        Map<ApplicationId, Set<JobType>> jobsByInstance = new HashMap<>();
         List<Run> activeRuns;
         while ( ! (activeRuns = this.jobs.active(applicationId)).isEmpty())
-            for (Run run : activeRuns)
+            for (Run run : activeRuns) {
+                Set<JobType> jobs = jobsByInstance.computeIfAbsent(run.id().application(), k -> new HashSet<>());
                 if (jobs.add(run.id().type())) {
-                    runJob(run.id().type());
+                    runJob(run.id().type(), run.id().application());
+                    if (multiInstance) {
+                        tester.outstandingChangeDeployer().run();
+                    }
                     triggerJobs();
-                }
-                else
+                } else {
                     throw new AssertionError("Job '" + run.id() + "' was run twice");
+                }
+            }
 
         assertFalse("Change should have no targets, but was " + instance().change(), instance().change().hasTargets());
         return this;
@@ -338,9 +350,14 @@ public class DeploymentContext {
         return runJob(JobType.from(tester.controller().system(), zone).get(), applicationPackage, null);
     }
 
-    /** Pulls the ready job trigger, and then runs the whole of the given job, successfully. */
+    /** Pulls the ready job trigger, and then runs the whole of the given job in the instance of this, successfully. */
     public DeploymentContext runJob(JobType type) {
-        var job = jobId(type);
+        return runJob(type, instanceId);
+    }
+
+    /** Pulls the ready job trigger, and then runs the whole of job for the given instance, successfully. */
+    public DeploymentContext runJob(JobType type, ApplicationId instance) {
+        var job = new JobId(instance, type);
         triggerJobs();
         doDeploy(job);
         if (job.type().isDeployment()) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
index 353756b3a4f..1f5fa243838 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/deployment/DeploymentTriggerTest.java
@@ -782,10 +782,9 @@ public class DeploymentTriggerTest {
                 .instances("instance1,instance2")
                 .region("us-east-3")
                 .build();
-        var app = tester.newDeploymentContext("tenant1", "application1", "instance1").submit(applicationPackage); // TODO jonmv: support instances in deployment context>
-        var otherInstance = tester.newDeploymentContext("tenant1", "application1", "instance2");
-        app.runJob(systemTest).runJob(stagingTest).runJob(productionUsEast3);
-        otherInstance.runJob(productionUsEast3);
+        var app = tester.newDeploymentContext("tenant1", "application1", "instance1")
+                        .submit(applicationPackage)
+                        .completeRollout();
         assertEquals(2, app.application().instances().size());
         assertEquals(2, app.application().productionDeployments().values().stream()
                            .mapToInt(Collection::size)
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
index f4475038b17..2e36b8969ba 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/RoutingPolicySerializerTest.java
@@ -7,10 +7,9 @@ import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicy;
 import com.yahoo.vespa.hosted.controller.routing.RoutingPolicyId;
-import com.yahoo.vespa.hosted.controller.routing.Status;
 import org.junit.Test;
 
 import java.time.Instant;
@@ -43,16 +42,16 @@ public class RoutingPolicySerializerTest {
                                                               Optional.of("zone1"),
                                                               instanceEndpoints,
                                                               applicationEndpoints,
-                                                              new Status(true, GlobalRouting.DEFAULT_STATUS)),
+                                                              new RoutingPolicy.Status(true, RoutingStatus.DEFAULT)),
                                        id2, new RoutingPolicy(id2,
                                                               HostName.from("long-and-ugly-name-2"),
                                                               Optional.empty(),
                                                               instanceEndpoints,
                                                               Set.of(),
-                                                              new Status(false,
-                                                                         new GlobalRouting(GlobalRouting.Status.out,
-                                                                                           GlobalRouting.Agent.tenant,
-                                                                                           Instant.ofEpochSecond(123)))));
+                                                              new RoutingPolicy.Status(false,
+                                                                                       new RoutingStatus(RoutingStatus.Value.out,
+                                                                                                         RoutingStatus.Agent.tenant,
+                                                                                                         Instant.ofEpochSecond(123)))));
         var serialized = serializer.fromSlime(owner, serializer.toSlime(policies));
         assertEquals(policies.size(), serialized.size());
         for (Iterator<RoutingPolicy> it1 = policies.values().iterator(), it2 = serialized.values().iterator(); it1.hasNext();) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java
index 4cc6bd93d01..234de233571 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/persistence/ZoneRoutingPolicySerializerTest.java
@@ -2,7 +2,7 @@
 package com.yahoo.vespa.hosted.controller.persistence;
 
 import com.yahoo.config.provision.zone.ZoneId;
-import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 import com.yahoo.vespa.hosted.controller.routing.ZoneRoutingPolicy;
 import org.junit.Test;
 
@@ -20,8 +20,8 @@ public class ZoneRoutingPolicySerializerTest {
         var serializer = new ZoneRoutingPolicySerializer(new RoutingPolicySerializer());
         var zone = ZoneId.from("prod", "us-north-1");
         var policy = new ZoneRoutingPolicy(zone,
-                                               GlobalRouting.status(GlobalRouting.Status.out, GlobalRouting.Agent.operator,
-                                                                    Instant.ofEpochMilli(123)));
+                                           RoutingStatus.create(RoutingStatus.Value.out, RoutingStatus.Agent.operator,
+                                                                Instant.ofEpochMilli(123)));
         var serialized = serializer.fromSlime(zone, serializer.toSlime(policy));
         assertEquals(policy, serialized);
     }
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
index e204a8f820d..e322205f064 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiTest.java
@@ -67,7 +67,7 @@ import com.yahoo.vespa.hosted.controller.notification.Notification;
 import com.yahoo.vespa.hosted.controller.notification.NotificationSource;
 import com.yahoo.vespa.hosted.controller.restapi.ContainerTester;
 import com.yahoo.vespa.hosted.controller.restapi.ControllerContainerTest;
-import com.yahoo.vespa.hosted.controller.routing.GlobalRouting;
+import com.yahoo.vespa.hosted.controller.routing.RoutingStatus;
 import com.yahoo.vespa.hosted.controller.security.AthenzCredentials;
 import com.yahoo.vespa.hosted.controller.security.AthenzTenantSpec;
 import com.yahoo.vespa.hosted.controller.support.access.SupportAccessGrant;
@@ -957,14 +957,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
                               new File("global-rotation-put.json"));
 
         // Status of routing policy is changed
-        assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
+        assertGlobalRouting(app.deploymentIdIn(westZone), RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
 
         // DELETE global rotation override status
         tester.assertResponse(request("/application/v4/tenant/tenant1/application/application1/instance/instance1/environment/prod/region/us-west-1/global-rotation/override", DELETE)
                                       .userIdentity(USER_ID)
                                       .data("{\"reason\":\"unit-test\"}"),
                               new File("global-rotation-delete.json"));
-        assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        assertGlobalRouting(app.deploymentIdIn(westZone), RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
 
         // SET global rotation override status by operator
         addUserToHostedOperatorRole(HostedAthenzIdentities.from(HOSTED_VESPA_OPERATOR));
@@ -972,7 +972,7 @@ public class ApplicationApiTest extends ControllerContainerTest {
                                       .userIdentity(HOSTED_VESPA_OPERATOR)
                                       .data("{\"reason\":\"unit-test\"}"),
                               new File("global-rotation-put.json"));
-        assertGlobalRouting(app.deploymentIdIn(westZone), GlobalRouting.Status.out, GlobalRouting.Agent.operator);
+        assertGlobalRouting(app.deploymentIdIn(westZone), RoutingStatus.Value.out, RoutingStatus.Agent.operator);
     }
 
     @Test
@@ -1865,14 +1865,14 @@ public class ApplicationApiTest extends ControllerContainerTest {
                 "Failed to deploy: Out of capacity");
     }
 
-    private void assertGlobalRouting(DeploymentId deployment, GlobalRouting.Status status, GlobalRouting.Agent agent) {
+    private void assertGlobalRouting(DeploymentId deployment, RoutingStatus.Value value, RoutingStatus.Agent agent) {
         var changedAt = tester.controller().clock().instant();
         var westPolicies = tester.controller().routing().policies().get(deployment);
         assertEquals(1, westPolicies.size());
         var westPolicy = westPolicies.values().iterator().next();
-        assertEquals(status, westPolicy.status().globalRouting().status());
-        assertEquals(agent, westPolicy.status().globalRouting().agent());
-        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().globalRouting().changedAt());
+        assertEquals(value, westPolicy.status().routingStatus().value());
+        assertEquals(agent, westPolicy.status().routingStatus().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), westPolicy.status().routingStatus().changedAt());
     }
 
     private static class RequestBuilder implements Supplier<Request> {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index e2ef27492af..afcc5e14d82 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -27,6 +27,7 @@ import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
 import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
+import com.yahoo.vespa.hosted.controller.application.TenantAndApplicationId;
 import com.yahoo.vespa.hosted.controller.application.pkg.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
@@ -165,8 +166,8 @@ public class RoutingPoliciesTest {
                      tester.policiesOf(context.instance().id()).size());
 
         // A zone in shared region is set out
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), RoutingStatus.Value.out,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
 
         // Weight of inactive zone is set to zero
@@ -176,16 +177,16 @@ public class RoutingPoliciesTest {
 
         // Other zone in shared region is set out. Entire record group for the region is removed as all zones in the
         // region are out (weight sum = 0)
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), RoutingStatus.Value.out,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L));
 
         // Everything is set back in
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone3), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone4), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone3), RoutingStatus.Value.in,
+                                                  RoutingStatus.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone4), RoutingStatus.Value.in,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 1L,
                                                                                            zone3, 1L,
@@ -480,8 +481,8 @@ public class RoutingPoliciesTest {
 
         // Global routing status is overridden in one zone
         var changedAt = tester.controllerTester().clock().instant();
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.out,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
 
         // Inactive zone is removed from global DNS record
@@ -490,15 +491,15 @@ public class RoutingPoliciesTest {
 
         // Status details is stored in policy
         var policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
-        assertEquals(GlobalRouting.Status.out, policy1.status().globalRouting().status());
-        assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent());
-        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
+        assertEquals(RoutingStatus.Value.out, policy1.status().routingStatus().value());
+        assertEquals(RoutingStatus.Agent.tenant, policy1.status().routingStatus().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().routingStatus().changedAt());
 
         // Other zone remains in
         var policy2 = tester.routingPolicies().get(context.deploymentIdIn(zone2)).values().iterator().next();
-        assertEquals(GlobalRouting.Status.in, policy2.status().globalRouting().status());
-        assertEquals(GlobalRouting.Agent.system, policy2.status().globalRouting().agent());
-        assertEquals(Instant.EPOCH, policy2.status().globalRouting().changedAt());
+        assertEquals(RoutingStatus.Value.in, policy2.status().routingStatus().value());
+        assertEquals(RoutingStatus.Agent.system, policy2.status().routingStatus().agent());
+        assertEquals(Instant.EPOCH, policy2.status().routingStatus().changedAt());
 
         // Next deployment does not affect status
         context.submit(applicationPackage).deferLoadBalancerProvisioningIn(Environment.prod).deploy();
@@ -509,15 +510,15 @@ public class RoutingPoliciesTest {
         // Deployment is set back in
         tester.controllerTester().clock().advance(Duration.ofHours(1));
         changedAt = tester.controllerTester().clock().instant();
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
         tester.assertTargets(context.instanceId(), EndpointId.of("r1"), 0, zone1, zone2);
 
         policy1 = tester.routingPolicies().get(context.deploymentIdIn(zone1)).values().iterator().next();
-        assertEquals(GlobalRouting.Status.in, policy1.status().globalRouting().status());
-        assertEquals(GlobalRouting.Agent.tenant, policy1.status().globalRouting().agent());
-        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().globalRouting().changedAt());
+        assertEquals(RoutingStatus.Value.in, policy1.status().routingStatus().value());
+        assertEquals(RoutingStatus.Agent.tenant, policy1.status().routingStatus().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), policy1.status().routingStatus().changedAt());
 
         // Deployment is set out through a new deployment.xml
         var applicationPackage2 = applicationPackageBuilder()
@@ -562,37 +563,37 @@ public class RoutingPoliciesTest {
         }
 
         // Set zone out
-        tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.out);
+        tester.routingPolicies().setRoutingStatus(zone2, RoutingStatus.Value.out);
         context1.flushDnsUpdates();
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
         for (var context : contexts) {
             var policies = tester.routingPolicies().get(context.instanceId());
-            assertTrue("Global routing status for policy remains " + GlobalRouting.Status.in,
+            assertTrue("Global routing status for policy remains " + RoutingStatus.Value.in,
                        policies.values().stream()
                                .map(RoutingPolicy::status)
-                               .map(Status::globalRouting)
-                               .map(GlobalRouting::status)
-                               .allMatch(status -> status == GlobalRouting.Status.in));
+                               .map(RoutingPolicy.Status::routingStatus)
+                               .map(RoutingStatus::value)
+                               .allMatch(status -> status == RoutingStatus.Value.in));
         }
         var changedAt = tester.controllerTester().clock().instant();
         var zonePolicy = tester.controllerTester().controller().curator().readZoneRoutingPolicy(zone2);
-        assertEquals(GlobalRouting.Status.out, zonePolicy.globalRouting().status());
-        assertEquals(GlobalRouting.Agent.operator, zonePolicy.globalRouting().agent());
-        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.globalRouting().changedAt());
+        assertEquals(RoutingStatus.Value.out, zonePolicy.routingStatus().value());
+        assertEquals(RoutingStatus.Agent.operator, zonePolicy.routingStatus().agent());
+        assertEquals(changedAt.truncatedTo(ChronoUnit.MILLIS), zonePolicy.routingStatus().changedAt());
 
         // Setting status per deployment does not affect status as entire zone is out
-        tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.in, GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
         context1.flushDnsUpdates();
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1);
 
         // Set single deployment out
-        tester.routingPolicies().setGlobalRoutingStatus(context1.deploymentIdIn(zone2), GlobalRouting.Status.out, GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context1.deploymentIdIn(zone2), RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
         context1.flushDnsUpdates();
 
         // Set zone back in. Deployment set explicitly out, remains out, the rest are in
-        tester.routingPolicies().setGlobalRoutingStatus(zone2, GlobalRouting.Status.in);
+        tester.routingPolicies().setRoutingStatus(zone2, RoutingStatus.Value.in);
         context1.flushDnsUpdates();
         tester.assertTargets(context1.instanceId(), EndpointId.defaultId(), 0, zone1);
         tester.assertTargets(context2.instanceId(), EndpointId.defaultId(), 0, zone1, zone2);
@@ -646,41 +647,41 @@ public class RoutingPoliciesTest {
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
 
         // Global routing status is overridden for one deployment
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.out,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone2);
 
         // Setting other deployment out implicitly sets all deployments in. Weight is set to zero, but that has no
         // impact on routing decisions when the weight sum is zero
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.out,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.out,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L));
 
         // One inactive deployment is put back in. Global DNS record now points to the only active deployment
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone1), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone1), RoutingStatus.Value.in,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
 
         // Setting zone (containing active deployment) out puts all deployments in
-        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.out);
+        tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.out);
         context.flushDnsUpdates();
-        assertEquals(GlobalRouting.Status.out, tester.routingPolicies().get(zone1).globalRouting().status());
+        assertEquals(RoutingStatus.Value.out, tester.routingPolicies().get(zone1).routingStatus().value());
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, ImmutableMap.of(zone1, 0L, zone2, 0L));
 
         // Setting zone back in removes the currently inactive deployment
-        tester.routingPolicies().setGlobalRoutingStatus(zone1, GlobalRouting.Status.in);
+        tester.routingPolicies().setRoutingStatus(zone1, RoutingStatus.Value.in);
         context.flushDnsUpdates();
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1);
 
         // Inactive deployment is set in
-        tester.routingPolicies().setGlobalRoutingStatus(context.deploymentIdIn(zone2), GlobalRouting.Status.in,
-                                                        GlobalRouting.Agent.tenant);
+        tester.routingPolicies().setRoutingStatus(context.deploymentIdIn(zone2), RoutingStatus.Value.in,
+                                                  RoutingStatus.Agent.tenant);
         context.flushDnsUpdates();
         for (var policy : tester.routingPolicies().get(context.instanceId()).values()) {
-            assertSame(GlobalRouting.Status.in, policy.status().globalRouting().status());
+            assertSame(RoutingStatus.Value.in, policy.status().routingStatus().value());
         }
         tester.assertTargets(context.instanceId(), EndpointId.of("r0"), 0, zone1, zone2);
     }
@@ -701,12 +702,103 @@ public class RoutingPoliciesTest {
                      records.get(0).data());
     }
 
+    @Test
+    public void application_endpoint_routing_policy() {
+        RoutingPoliciesTester tester = new RoutingPoliciesTester();
+        TenantAndApplicationId application = TenantAndApplicationId.from("tenant1", "app1");
+        ApplicationId betaInstance = application.instance("beta");
+        ApplicationId mainInstance = application.instance("main");
+
+        DeploymentContext betaContext = tester.newDeploymentContext(betaInstance);
+        DeploymentContext mainContext = tester.newDeploymentContext(mainInstance);
+        var applicationPackage = applicationPackageBuilder()
+                .instances("beta,main")
+                .region(zone1.region())
+                .region(zone2.region())
+                .applicationEndpoint("a0", "c0", "us-west-1",
+                                     Map.of(betaInstance.instance(), 2,
+                                            mainInstance.instance(), 8))
+                .applicationEndpoint("a1", "c1", "us-central-1",
+                                     Map.of(betaInstance.instance(), 4,
+                                            mainInstance.instance(), 6))
+                .build();
+        for (var zone : List.of(zone1, zone2)) {
+            tester.provisionLoadBalancers(2, betaInstance, zone);
+            tester.provisionLoadBalancers(2, mainInstance, zone);
+        }
+
+        // Deploy both instances
+        betaContext.submit(applicationPackage).deploy();
+
+        // Application endpoint points to both instances with correct weights
+        DeploymentId betaZone1 = betaContext.deploymentIdIn(zone1);
+        DeploymentId mainZone1 = mainContext.deploymentIdIn(zone1);
+        DeploymentId betaZone2 = betaContext.deploymentIdIn(zone2);
+        DeploymentId mainZone2 = mainContext.deploymentIdIn(zone2);
+        tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
+                             Map.of(betaZone1, 2,
+                                    mainZone1, 8));
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 4,
+                                    mainZone2, 6));
+
+        // Weights are updated
+        applicationPackage = applicationPackageBuilder()
+                .instances("beta,main")
+                .region(zone1.region())
+                .region(zone2.region())
+                .applicationEndpoint("a0", "c0", "us-west-1",
+                                     Map.of(betaInstance.instance(), 3,
+                                            mainInstance.instance(), 7))
+                .applicationEndpoint("a1", "c1", "us-central-1",
+                                     Map.of(betaInstance.instance(), 1,
+                                            mainInstance.instance(), 9))
+                .build();
+        betaContext.submit(applicationPackage).deploy();
+        tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
+                             Map.of(betaZone1, 3,
+                                    mainZone1, 7));
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 1,
+                                    mainZone2, 9));
+
+        // Changing routing status updates weight
+        tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.out, RoutingStatus.Agent.tenant);
+        betaContext.flushDnsUpdates();
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 1,
+                                    mainZone2, 0));
+        tester.routingPolicies().setRoutingStatus(mainZone2, RoutingStatus.Value.in, RoutingStatus.Agent.tenant);
+        betaContext.flushDnsUpdates();
+        tester.assertTargets(application, EndpointId.of("a1"), ClusterSpec.Id.from("c1"), 1,
+                             Map.of(betaZone2, 1,
+                                    mainZone2, 9));
+
+        // An endpoint is removed
+        applicationPackage = applicationPackageBuilder()
+                .instances("beta,main")
+                .region(zone1.region())
+                .region(zone2.region())
+                .applicationEndpoint("a0", "c0", "us-west-1",
+                                     Map.of(betaInstance.instance(), 1))
+                .build();
+        betaContext.submit(applicationPackage).deploy();
+
+        // Application endpoints now point to a single instance
+        tester.assertTargets(application, EndpointId.of("a0"), ClusterSpec.Id.from("c0"), 0,
+                             Map.of(betaZone1, 1));
+        assertTrue("Endpoint removed",
+                   tester.controllerTester().controller().routing()
+                         .endpointsOf(application)
+                         .named(EndpointId.of("a1")).isEmpty());
+    }
+
     /** Returns an application package builder that satisfies requirements for a directly routed endpoint */
     private static ApplicationPackageBuilder applicationPackageBuilder() {
-        return new ApplicationPackageBuilder()
-                .athenzIdentity(AthenzDomain.from("domain"), AthenzService.from("service"));
+        return new ApplicationPackageBuilder().athenzIdentity(AthenzDomain.from("domain"),
+                                                              AthenzService.from("service"));
     }
-    
+
     private static List<LoadBalancer> createLoadBalancers(ZoneId zone, ApplicationId application, boolean shared, int count) {
         List<LoadBalancer> loadBalancers = new ArrayList<>();
         for (int i = 0; i < count; i++) {
@@ -819,7 +911,35 @@ public class RoutingPoliciesTest {
                          .collect(Collectors.toList());
         }
 
-        private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster, int loadBalancerId, Map<ZoneId, Long> zoneWeights) {
+        /** Assert that an application endpoint points to given targets and weights */
+        private void assertTargets(TenantAndApplicationId application, EndpointId endpointId, ClusterSpec.Id cluster,
+                                   int loadBalancerId, Map<DeploymentId, Integer> deploymentWeights) {
+            Map<String, List<DeploymentId>> deploymentsByDnsName = new HashMap<>();
+            for (var deployment : deploymentWeights.keySet()) {
+                EndpointList applicationEndpoints = tester.controller().routing().endpointsOf(application)
+                                                          .named(endpointId)
+                                                          .targets(deployment)
+                                                          .cluster(cluster);
+                assertEquals("Expected a single endpoint with ID '" + endpointId + "'", 1,
+                             applicationEndpoints.size());
+                String dnsName = applicationEndpoints.asList().get(0).dnsName();
+                deploymentsByDnsName.computeIfAbsent(dnsName, (k) -> new ArrayList<>())
+                                    .add(deployment);
+            }
+            assertEquals("Found " + endpointId + " for " + application, 1, deploymentsByDnsName.size());
+            deploymentsByDnsName.forEach((dnsName, deployments) -> {
+                Set<String> weightedTargets = deployments.stream()
+                                                           .map(d -> "weighted/lb-" + loadBalancerId + "--" +
+                                                                     d.applicationId().serializedForm() + "--" + d.zoneId().value() +
+                                                                     "/dns-zone-1/" + d.zoneId().value() + "/" + deploymentWeights.get(d))
+                                                           .collect(Collectors.toSet());
+                assertEquals(dnsName + " has expected targets", weightedTargets, aliasDataOf(dnsName));
+            });
+        }
+
+        /** Assert that an instance endpoint points to given targets and weights */
+        private void assertTargets(ApplicationId instance, EndpointId endpointId, ClusterSpec.Id cluster,
+                                   int loadBalancerId, Map<ZoneId, Long> zoneWeights) {
             Set<String> latencyTargets = new HashSet<>();
             Map<String, List<ZoneId>> zonesByRegionEndpoint = new HashMap<>();
             for (var zone : zoneWeights.keySet()) {