Support filtering endpoints by target zone(s)

6 files changed, 90 insertions, 57 deletions

diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
index 9efb745ddf6..31812b9346b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/RoutingController.java
@@ -103,8 +103,6 @@ public class RoutingController {
     }
 
     /** Returns global-scoped endpoints for given instance */
-    // TODO(mpolden): Add a endpointsOf(Instance, DeploymentId) variant of this that only returns global endpoint of
-    //                which deployment is a member
     public EndpointList endpointsOf(Application application, InstanceName instanceName) {
         var endpoints = new LinkedHashSet<Endpoint>();
         // Add global endpoints provided by rotations
@@ -113,8 +111,9 @@ public class RoutingController {
             var deployments = rotation.regions().stream()
                                       .map(region -> new DeploymentId(instance.id(), ZoneId.from(Environment.prod, region)))
                                       .collect(Collectors.toList());
-            EndpointList.global(RoutingId.of(instance.id(), rotation.endpointId()),
-                                controller.system(), routingMethodsOfAll(deployments, application))
+            var targets = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList());
+            EndpointList.global(RoutingId.of(instance.id(), rotation.endpointId()), controller.system(), targets,
+                                routingMethodsOfAll(deployments, application))
                         .requiresRotation()
                         .forEach(endpoints::add);
         }
@@ -129,7 +128,8 @@ public class RoutingController {
             }
         }
         deploymentsByRoutingId.forEach((routingId, deployments) -> {
-            EndpointList.global(routingId, controller.system(), routingMethodsOfAll(deployments, application))
+            var targets = deployments.stream().map(DeploymentId::zoneId).collect(Collectors.toList());
+            EndpointList.global(routingId, controller.system(), targets, routingMethodsOfAll(deployments, application))
                         .not().requiresRotation()
                         .forEach(endpoints::add);
         });
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
index 31394ef8d33..e5848055603 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/Endpoint.java
@@ -9,6 +9,7 @@ import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.api.identifiers.DeploymentId;
 
 import java.net.URI;
+import java.util.List;
 import java.util.Objects;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
@@ -29,22 +30,29 @@ public class Endpoint {
 
     private final String name;
     private final URI url;
+    private final List<ZoneId> zones;
     private final Scope scope;
     private final boolean legacy;
     private final RoutingMethod routingMethod;
     private final boolean tls;
     private final boolean wildcard;
 
-    private Endpoint(String name, ApplicationId application, ZoneId zone, SystemName system, Port port, boolean legacy,
-                     RoutingMethod routingMethod, boolean wildcard) {
+    private Endpoint(String name, ApplicationId application, List<ZoneId> zones, Scope scope, SystemName system, Port port,
+                     boolean legacy, RoutingMethod routingMethod, boolean wildcard) {
         Objects.requireNonNull(name, "name must be non-null");
         Objects.requireNonNull(application, "application must be non-null");
+        Objects.requireNonNull(zones, "zones must be non-null");
+        Objects.requireNonNull(scope, "scope must be non-null");
         Objects.requireNonNull(system, "system must be non-null");
         Objects.requireNonNull(port, "port must be non-null");
         Objects.requireNonNull(routingMethod, "routingMethod must be non-null");
+        if (scope == Scope.zone && zones.size() != 1) {
+            throw new IllegalArgumentException("A single zone must be given for zone-scoped endpoints");
+        }
         this.name = name;
-        this.url = createUrl(name, application, zone, system, port, legacy, routingMethod);
-        this.scope = zone == null ? Scope.global : Scope.zone;
+        this.url = createUrl(name, application, zones, scope, system, port, legacy, routingMethod);
+        this.zones = List.copyOf(zones);
+        this.scope = scope;
         this.legacy = legacy;
         this.routingMethod = routingMethod;
         this.tls = port.tls;
@@ -73,6 +81,11 @@ public class Endpoint {
         return url.getAuthority().replaceAll(":.*", "");
     }
 
+    /** Returns the zone(s) to which this routes traffic */
+    public List<ZoneId> zones() {
+        return zones;
+    }
+
     /** Returns the scope of this */
     public Scope scope() {
         return scope;
@@ -133,8 +146,8 @@ public class Endpoint {
         return dnsSuffix(system, false);
     }
 
-    private static URI createUrl(String name, ApplicationId application, ZoneId zone, SystemName system,
-                                 Port port, boolean legacy, RoutingMethod routingMethod) {
+    private static URI createUrl(String name, ApplicationId application, List<ZoneId> zones, Scope scope,
+                                 SystemName system, Port port, boolean legacy, RoutingMethod routingMethod) {
         String scheme = port.tls ? "https" : "http";
         String separator = separator(system, routingMethod, port.tls);
         String portPart = port.isDefault() ? "" : ":" + port.port;
@@ -146,7 +159,7 @@ public class Endpoint {
                           separator +
                           sanitize(application.tenant().value()) +
                           "." +
-                          scopePart(zone, legacy) +
+                          scopePart(scope, zones, legacy) +
                           dnsSuffix(system, legacy) +
                           portPart +
                           "/");
@@ -168,8 +181,9 @@ public class Endpoint {
         return name + separator;
     }
 
-    private static String scopePart(ZoneId zone, boolean legacy) {
-        if (zone == null) return "global";
+    private static String scopePart(Scope scope, List<ZoneId> zones, boolean legacy) {
+        if (scope == Scope.global) return "global";
+        var zone = zones.get(0);
         if (!legacy && zone.environment().isProduction()) return zone.region().value(); // Skip prod environment for non-legacy endpoints
         return zone.region().value() + "." + zone.environment().value();
     }
@@ -283,7 +297,8 @@ public class Endpoint {
 
         private final ApplicationId application;
 
-        private ZoneId zone;
+        private Scope scope;
+        private List<ZoneId> zones;
         private ClusterSpec.Id cluster;
         private EndpointId endpointId;
         private Port port;
@@ -301,35 +316,44 @@ public class Endpoint {
                 throw new IllegalArgumentException("Cannot set multiple target types");
             }
             this.cluster = cluster;
-            this.zone = zone;
+            this.scope = Scope.zone;
+            this.zones = List.of(zone);
             return this;
         }
 
         /** Sets the endpoint target ID for this (as defined in deployments.xml) */
         public EndpointBuilder named(EndpointId endpointId) {
+           return named(endpointId, List.of());
+        }
+
+        /** Sets the endpoint ID for this (as defined in deployments.xml) */
+        public EndpointBuilder named(EndpointId endpointId, List<ZoneId> targets) {
             if (cluster != null || wildcard) {
                 throw new IllegalArgumentException("Cannot set multiple target types");
             }
             this.endpointId = endpointId;
+            this.zones = targets;
+            this.scope = Scope.global;
             return this;
         }
 
         /** Sets the global wildcard target for this */
         public EndpointBuilder wildcard() {
-            if (endpointId != null || cluster != null) {
-                throw new IllegalArgumentException("Cannot set multiple target types");
-            }
-            this.wildcard = true;
-            return this;
+            return wildcard(Scope.global, List.of());
         }
 
         /** Sets the zone wildcard target for this */
         public EndpointBuilder wildcard(ZoneId zone) {
-            if(endpointId != null || cluster != null) {
+            return wildcard(Scope.zone, List.of(zone));
+        }
+
+        private EndpointBuilder wildcard(Scope scope, List<ZoneId> zones) {
+            if (endpointId != null || cluster != null) {
                 throw new IllegalArgumentException("Cannot set multiple target types");
             }
-            this.zone = zone;
             this.wildcard = true;
+            this.scope = scope;
+            this.zones = zones;
             return this;
         }
 
@@ -369,7 +393,7 @@ public class Endpoint {
             if (routingMethod.isDirect() && !port.isDefault()) {
                 throw new IllegalArgumentException("Routing method " + routingMethod + " can only use default port");
             }
-            return new Endpoint(name, application, zone, system, port, legacy, routingMethod, wildcard);
+            return new Endpoint(name, application, zones, scope, system, port, legacy, routingMethod, wildcard);
         }
 
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
index a4c5570ee3f..a4c7a57247c 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/EndpointList.java
@@ -4,6 +4,7 @@ package com.yahoo.vespa.hosted.controller.application;
 import com.yahoo.collections.AbstractFilteringList;
 import com.yahoo.config.provision.SystemName;
 import com.yahoo.config.provision.zone.RoutingMethod;
+import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.application.Endpoint.Port;
 import com.yahoo.vespa.hosted.controller.routing.RoutingId;
 
@@ -41,6 +42,16 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
         return matching(endpoint -> endpoint.name().equals(id.id()));
     }
 
+    /** Returns the subset of endpoints which target all of the given zones */
+    public EndpointList targets(List<ZoneId> zones) {
+        return matching(endpoint -> endpoint.zones().containsAll(zones));
+    }
+
+    /** Returns the subset of endpoints which target the given zones */
+    public EndpointList targets(ZoneId zone) {
+        return targets(List.of(zone));
+    }
+
     /** Returns the subset of endpoints that are considered legacy */
     public EndpointList legacy() {
         return matching(Endpoint::legacy);
@@ -57,7 +68,7 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
     }
 
     /** Returns all global endpoints for given routing ID and system provided by given routing methods */
-    public static EndpointList global(RoutingId routingId, SystemName system, List<RoutingMethod> routingMethods) {
+    public static EndpointList global(RoutingId routingId, SystemName system, List<ZoneId> targets, List<RoutingMethod> routingMethods) {
         var endpoints = new ArrayList<Endpoint>();
         var directMethods = 0;
         for (var method : routingMethods) {
@@ -66,20 +77,20 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
                                                    "direct methods, got " + routingMethods);
             }
             endpoints.add(Endpoint.of(routingId.application())
-                                  .named(routingId.endpointId())
+                                  .named(routingId.endpointId(), targets)
                                   .on(Port.fromRoutingMethod(method))
                                   .routingMethod(method)
                                   .in(system));
             // TODO(mpolden): Remove this once all applications have migrated away from legacy endpoints
             if (method == RoutingMethod.shared) {
                 endpoints.add(Endpoint.of(routingId.application())
-                                      .named(routingId.endpointId())
+                                      .named(routingId.endpointId(), targets)
                                       .on(Port.plain(4080))
                                       .legacy()
                                       .routingMethod(method)
                                       .in(system));
                 endpoints.add(Endpoint.of(routingId.application())
-                                      .named(routingId.endpointId())
+                                      .named(routingId.endpointId(), targets)
                                       .on(Port.tls(4443))
                                       .legacy()
                                       .routingMethod(method)
@@ -89,10 +100,6 @@ public class EndpointList extends AbstractFilteringList<Endpoint, EndpointList>
         return new EndpointList(endpoints);
     }
 
-    public static EndpointList global(RoutingId routingId, SystemName system, RoutingMethod routingMethod) {
-        return global(routingId, system, List.of(routingMethod));
-    }
-
     public static EndpointList copyOf(Collection<Endpoint> endpoints) {
         return new EndpointList(endpoints);
     }
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
index 8c2b2d043d4..be3f4e50dc7 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/restapi/application/ApplicationApiHandler.java
@@ -1070,11 +1070,12 @@ public class ApplicationApiHandler extends LoggingRequestHandler {
             }
         }
         // Add global endpoints
-        if (deploymentId.zoneId().environment().isProduction()) { // Global endpoints can only point to production deployments
-            for (var endpoint : controller.routing().endpointsOf(application, deploymentId.applicationId().instance()).not().legacy()) {
-                // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level.
-                toSlime(endpoint, "", endpointArray.addObject());
-            }
+        var globalEndpoints = controller.routing().endpointsOf(application, deploymentId.applicationId().instance())
+                                        .not().legacy()
+                                        .targets(deploymentId.zoneId());
+        for (var endpoint : globalEndpoints) {
+            // TODO(mpolden): Pass cluster name. Cluster that a global endpoint points to is not available at this level.
+            toSlime(endpoint, "", endpointArray.addObject());
         }
         // TODO(mpolden): Remove this once all clients stop reading it
         Cursor serviceUrlArray = response.setArray("serviceUrls");
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
index 0bcf25dd4b8..468ecc424a9 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepositoryTest.java
@@ -6,12 +6,10 @@ import com.yahoo.config.provision.zone.RoutingMethod;
 import com.yahoo.vespa.hosted.controller.ControllerTester;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
 import com.yahoo.vespa.hosted.controller.application.AssignedRotation;
-import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentTester;
-import com.yahoo.vespa.hosted.controller.routing.RoutingId;
+import com.yahoo.vespa.hosted.controller.integration.ZoneApiMock;
 import com.yahoo.vespa.hosted.rotation.config.RotationsConfig;
 import org.junit.Before;
 import org.junit.Rule;
@@ -71,8 +69,7 @@ public class RotationRepositoryTest {
 
         assertEquals(List.of(expected.id()), rotationIds(application.instance().rotations()));
         assertEquals(URI.create("https://app1--tenant1.global.vespa.oath.cloud:4443/"),
-                     EndpointList.global(RoutingId.of(application.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared)
-                                 .primary().get().url());
+                     tester.controller().routing().endpointsOf(application.instanceId()).primary().get().url());
         try (RotationLock lock = repository.lock()) {
             List<AssignedRotation> rotations = repository.getOrAssignRotations(application.application().deploymentSpec(),
                                                                                application.instance(),
@@ -141,15 +138,19 @@ public class RotationRepositoryTest {
     public void prefixes_system_when_not_main() {
         ApplicationPackage applicationPackage = new ApplicationPackageBuilder()
                 .globalServiceId("foo")
-                .region("us-east-3")
-                .region("us-west-1")
+                .region("cd-us-central-1")
+                .region("cd-us-west-1")
                 .build();
+        var zones = List.of(ZoneApiMock.fromId("prod.cd-us-central-1"), ZoneApiMock.fromId("prod.cd-us-west-1"));
+        tester.controllerTester().zoneRegistry()
+              .setZones(zones)
+              .setRoutingMethod(zones, RoutingMethod.shared)
+              .setSystemName(SystemName.cd);
         var application2 = tester.newDeploymentContext("tenant2", "app2", "default");
         application2.submit(applicationPackage);
         assertEquals(List.of(new RotationId("foo-1")), rotationIds(application2.instance().rotations()));
         assertEquals("https://cd--app2--tenant2.global.vespa.oath.cloud:4443/",
-                     EndpointList.global(RoutingId.of(application2.instanceId(), EndpointId.defaultId()), SystemName.cd, RoutingMethod.shared)
-                                 .primary().get().url().toString());
+                     tester.controller().routing().endpointsOf(application2.instanceId()).primary().get().url().toString());
     }
 
     @Test
@@ -165,11 +166,9 @@ public class RotationRepositoryTest {
         assertEquals(List.of(new RotationId("foo-1")), rotationIds(instance1.instance().rotations()));
         assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations()));
         assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     EndpointList.global(RoutingId.of(instance1.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared)
-                                 .primary().get().url());
+                     tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url());
         assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     EndpointList.global(RoutingId.of(instance2.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared)
-                                 .primary().get().url());
+                     tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url());
     }
 
     @Test
@@ -187,11 +186,9 @@ public class RotationRepositoryTest {
         assertEquals(List.of(new RotationId("foo-2")), rotationIds(instance2.instance().rotations()));
 
         assertEquals(URI.create("https://instance1--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     EndpointList.global(RoutingId.of(instance1.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared)
-                                 .primary().get().url());
+                     tester.controller().routing().endpointsOf(instance1.instanceId()).primary().get().url());
         assertEquals(URI.create("https://instance2--application1--tenant1.global.vespa.oath.cloud:4443/"),
-                     EndpointList.global(RoutingId.of(instance2.instanceId(), EndpointId.defaultId()), SystemName.main, RoutingMethod.shared)
-                                 .primary().get().url());
+                     tester.controller().routing().endpointsOf(instance2.instanceId()).primary().get().url());
     }
 
     private void assertSingleRotation(Rotation expected, List<AssignedRotation> assignedRotations, RotationRepository repository) {
diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
index 719b2cc47f9..e44a1364185 100644
--- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
+++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/routing/RoutingPoliciesTest.java
@@ -24,8 +24,8 @@ import com.yahoo.vespa.hosted.controller.api.integration.dns.Record;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordData;
 import com.yahoo.vespa.hosted.controller.api.integration.dns.RecordName;
 import com.yahoo.vespa.hosted.controller.application.ApplicationPackage;
+import com.yahoo.vespa.hosted.controller.application.Endpoint;
 import com.yahoo.vespa.hosted.controller.application.EndpointId;
-import com.yahoo.vespa.hosted.controller.application.EndpointList;
 import com.yahoo.vespa.hosted.controller.application.SystemApplication;
 import com.yahoo.vespa.hosted.controller.deployment.ApplicationPackageBuilder;
 import com.yahoo.vespa.hosted.controller.deployment.DeploymentContext;
@@ -659,8 +659,12 @@ public class RoutingPoliciesTest {
         }
 
         private void assertTargets(ApplicationId application, EndpointId endpointId, int loadBalancerId, ZoneId ...zone) {
-            var endpoint = EndpointList.global(RoutingId.of(application, endpointId), tester.controller().system(), RoutingMethod.exclusive)
-                                       .primary().get().dnsName();
+            var endpoint = tester.controller().routing().endpointsOf(application)
+                                 .named(endpointId)
+                                 .targets(List.of(zone))
+                                 .primary()
+                                 .map(Endpoint::dnsName)
+                                 .orElse("<none>");
             var zoneTargets = Arrays.stream(zone)
                                     .map(z -> "lb-" + loadBalancerId + "--" + application.serializedForm() + "--" +
                                               z.value() + "/dns-zone-1/" + z.value())