Support application-level endpoints in DeploymentSpec

8 files changed, 335 insertions, 107 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
index 9d674d43fda..23a5dce3c25 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
@@ -1,14 +1,12 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.application.api;
 
-import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.AthenzService;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 
 import java.time.Instant;
-import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Objects;
@@ -51,7 +49,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
         this.globalServiceId = globalServiceId;
         this.athenzService = athenzService;
         this.notifications = notifications;
-        this.endpoints = List.copyOf(validateEndpoints(endpoints, steps()));
+        this.endpoints = List.copyOf(endpoints);
         validateZones(new HashSet<>(), new HashSet<>(), this);
         validateEndpoints(steps(), globalServiceId, this.endpoints);
     }
@@ -91,31 +89,6 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
         }
     }
 
-    /** Validates the endpoints and makes sure default values are respected */
-    private List<Endpoint> validateEndpoints(List<Endpoint> endpoints, List<DeploymentSpec.Step> steps) {
-        Objects.requireNonNull(endpoints, "Missing endpoints parameter");
-
-        var productionRegions = steps.stream()
-                                     .filter(step -> step.concerns(Environment.prod))
-                                     .flatMap(step -> step.zones().stream())
-                                     .flatMap(zone -> zone.region().stream())
-                                     .map(RegionName::value)
-                                     .collect(Collectors.toSet());
-
-        var rebuiltEndpointsList = new ArrayList<Endpoint>();
-
-        for (var endpoint : endpoints) {
-            if (endpoint.regions().isEmpty()) {
-                var rebuiltEndpoint = endpoint.withRegions(productionRegions);
-                rebuiltEndpointsList.add(rebuiltEndpoint);
-            } else {
-                rebuiltEndpointsList.add(endpoint);
-            }
-        }
-
-        return List.copyOf(rebuiltEndpointsList);
-    }
-
     /** Throw an IllegalArgumentException if an endpoint refers to a region that is not declared in 'prod' */
     private void validateEndpoints(List<DeploymentSpec.Step> steps, Optional<String> globalServiceId, List<Endpoint> endpoints) {
         if (globalServiceId.isPresent() && ! endpoints.isEmpty()) {
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
index 9562584bc0a..97ece3a675e 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
@@ -11,7 +11,6 @@ import com.yahoo.config.provision.RegionName;
 
 import java.io.Reader;
 import java.time.Duration;
-import java.util.ArrayList;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
@@ -39,6 +38,7 @@ public class DeploymentSpec {
                                                                   Optional.empty(),
                                                                   Optional.empty(),
                                                                   Optional.empty(),
+                                                                  List.of(),
                                                                   "<deployment version='1.0'/>");
 
     private final List<Step> steps;
@@ -47,6 +47,7 @@ public class DeploymentSpec {
     private final Optional<Integer> majorVersion;
     private final Optional<AthenzDomain> athenzDomain;
     private final Optional<AthenzService> athenzService;
+    private final List<Endpoint> endpoints;
 
     private final String xmlForm;
 
@@ -54,23 +55,26 @@ public class DeploymentSpec {
                           Optional<Integer> majorVersion,
                           Optional<AthenzDomain> athenzDomain,
                           Optional<AthenzService> athenzService,
+                          List<Endpoint> endpoints,
                           String xmlForm) {
-        this.steps = List.copyOf(steps);
-        this.majorVersion = majorVersion;
-        this.athenzDomain = athenzDomain;
-        this.athenzService = athenzService;
-        this.xmlForm = xmlForm;
+        this.steps = List.copyOf(Objects.requireNonNull(steps));
+        this.majorVersion = Objects.requireNonNull(majorVersion);
+        this.athenzDomain = Objects.requireNonNull(athenzDomain);
+        this.athenzService = Objects.requireNonNull(athenzService);
+        this.xmlForm = Objects.requireNonNull(xmlForm);
+        this.endpoints = List.copyOf(Objects.requireNonNull(endpoints));
         validateTotalDelay(steps);
         validateUpgradePoliciesOfIncreasingConservativeness(steps);
         validateAthenz();
+        validateApplicationEndpoints();
     }
 
     /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */
     private void validateTotalDelay(List<Step> steps) {
         long totalDelaySeconds = steps.stream().mapToLong(step -> (step.delay().getSeconds())).sum();
         if (totalDelaySeconds > Duration.ofHours(48).getSeconds())
-            throw new IllegalArgumentException("The total delay specified is " + Duration.ofSeconds(totalDelaySeconds) +
-                                               " but max 48 hours is allowed");
+            illegal("The total delay specified is " + Duration.ofSeconds(totalDelaySeconds) +
+                    " but max 48 hours is allowed");
     }
 
     /** Throws an IllegalArgumentException if any instance has a looser upgrade policy than the previous */
@@ -81,8 +85,8 @@ public class DeploymentSpec {
             List<DeploymentInstanceSpec> specs = instances(List.of(step));
             for (DeploymentInstanceSpec spec : specs) {
                 if (spec.upgradePolicy().compareTo(previous) < 0)
-                    throw new IllegalArgumentException("Instance '" + spec.name() + "' cannot have a looser upgrade " +
-                                                       "policy than the previous of '" + previous + "'");
+                    illegal("Instance '" + spec.name() + "' cannot have a looser upgrade " +
+                            "policy than the previous of '" + previous + "'");
 
                 strictest = Comparables.max(strictest, spec.upgradePolicy());
             }
@@ -101,7 +105,7 @@ public class DeploymentSpec {
             for (DeploymentInstanceSpec instance : instances()) {
                 for (DeploymentSpec.DeclaredZone zone : instance.zones()) {
                     if (zone.athenzService().isPresent()) {
-                        throw new IllegalArgumentException("Athenz service configured for zone: " + zone + ", but Athenz domain is not configured");
+                        illegal("Athenz service configured for zone: " + zone + ", but Athenz domain is not configured");
                     }
                 }
             }
@@ -111,13 +115,29 @@ public class DeploymentSpec {
             for (DeploymentInstanceSpec instance : instances()) {
                 for (DeploymentSpec.DeclaredZone zone : instance.zones()) {
                     if (zone.athenzService().isEmpty()) {
-                        throw new IllegalArgumentException("Athenz domain is configured, but Athenz service not configured for zone: " + zone);
+                        illegal("Athenz domain is configured, but Athenz service not configured for zone: " + zone);
                     }
                 }
             }
         }
     }
 
+    private void validateApplicationEndpoints() {
+        for (var endpoint : endpoints) {
+            if (endpoint.level() != Endpoint.Level.application) illegal("Endpoint '" + endpoint.endpointId() + "' must be an application–level endpoint, got " + endpoint.level());
+            String prefix = "Application-level endpoint '" + endpoint.endpointId() + "': ";
+            for (var target : endpoint.targets()) {
+                Optional<DeploymentInstanceSpec> instance = instance(target.instance());
+                if (instance.isEmpty()) {
+                    illegal(prefix + "targets undeclared instance '" + target.instance() + "'");
+                }
+                if (!instance.get().deploysTo(Environment.prod, target.region())) {
+                    illegal(prefix + "targets undeclared region '" + target.region() +
+                            "' in instance '" + target.instance() + "'");
+                }
+            }
+        }
+    }
 
     /** Returns the major version this application is pinned to, or empty (default) to allow all major versions */
     public Optional<Integer> majorVersion() { return majorVersion; }
@@ -176,6 +196,11 @@ public class DeploymentSpec {
         return instances(steps);
     }
 
+    /** Returns the application-level endpoints of this, if any */
+    public List<Endpoint> endpoints() {
+        return endpoints;
+    }
+
     private static List<DeploymentInstanceSpec> instances(List<DeploymentSpec.Step> steps) {
         return steps.stream()
                     .flatMap(DeploymentSpec::flatten)
@@ -187,6 +212,11 @@ public class DeploymentSpec {
         return step.steps().stream().flatMap(DeploymentSpec::flatten);
     }
 
+
+    private static void illegal(String message) {
+        throw new IllegalArgumentException(message);
+    }
+
     /**
      * Creates a deployment spec from XML.
      *
@@ -317,9 +347,9 @@ public class DeploymentSpec {
         public DeclaredZone(Environment environment, Optional<RegionName> region, boolean active,
                             Optional<AthenzService> athenzService, Optional<String> testerFlavor) {
             if (environment != Environment.prod && region.isPresent())
-                throw new IllegalArgumentException("Non-prod environments cannot specify a region");
+                illegal("Non-prod environments cannot specify a region");
             if (environment == Environment.prod && region.isEmpty())
-                throw new IllegalArgumentException("Prod environments must be specified with a region");
+                illegal("Prod environments must be specified with a region");
             this.environment = environment;
             this.region = region;
             this.active = active;
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java
index b3d2aa5e2a1..78bf9fc68c4 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Endpoint.java
@@ -1,21 +1,22 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.application.api;
 
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 
+import java.util.List;
 import java.util.Objects;
-import java.util.Set;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 /**
- * Represents a (global) endpoint in 'deployments.xml'.  It contains the name of the
- * endpoint (endpointId) and the name of the container cluster that the endpoint
- * should point to.
+ * Represents an application- or instance-level endpoint in deployments.xml.
  *
- * If the endpoint is not set it will default to the string "default".
+ * - An instance-level endpoint is global and can span multiple regions within a single instance.
+ * - An application-level endpoint points can span multiple instances within a single region.
  *
  * @author ogronnesby
+ * @author mpolden
  */
 public class Endpoint {
 
@@ -34,35 +35,67 @@ public class Endpoint {
 
     private final String endpointId;
     private final String containerId;
-    private final Set<RegionName> regions;
+    private final Level level;
+    private final List<Target> targets;
 
-    public Endpoint(String endpointId, String containerId, Set<String> regions) {
+    public Endpoint(String endpointId, String containerId, Level level, List<Target> targets) {
         this.endpointId = Objects.requireNonNull(endpointId, "endpointId must be non-null");
         this.containerId = Objects.requireNonNull(containerId, "containerId must be non-null");
-        this.regions = Set.copyOf(
-                Objects.requireNonNull(
-                        regions.stream().map(RegionName::from).collect(Collectors.toList()),
-                        "Missing 'regions' parameter"));
-
+        this.level = Objects.requireNonNull(level, "level must be non-null");
+        this.targets = List.copyOf(Objects.requireNonNull(targets, "targets must be non-null"));
         if (endpointId().length() > endpointMaxLength || !endpointPattern.matcher(endpointId()).matches()) {
             throw new IllegalArgumentException("Invalid endpoint ID: '" + endpointId() + "'");
         }
+        if (targets.isEmpty()) throw new IllegalArgumentException("targets must be non-empty");
+        for (int i = 0; i < targets.size(); i++) {
+            for (int j = 0; j < i; j++) {
+                Target a = targets.get(i);
+                Target b = targets.get(j);
+                if (level == Level.application) {
+                    // - All instance names must be distinct
+                    // - All region names must be equal
+                    if (a.instance().equals(b.instance())) throw new IllegalArgumentException("Instance '" + a.instance +
+                                                                                              "' declared multiple times, but allowed at most once");
+                    if (!a.region().equals(b.region())) throw new IllegalArgumentException("Instance '" + a.instance + "' declares a region different from instance '" +
+                                                                                           b.instance() + "': '" + a.region() + "'");
+                }
+                if (level == Level.instance && a.region.equals(b.region)) {
+                    // - Instance name is implicit
+                    // - All regions must be distinct
+                    throw new IllegalArgumentException("Region '" + a.region + "' declared multiple times, but allowed at most once");
+                }
+            }
+        }
     }
 
+    /** The unique identifer of this */
     public String endpointId() {
         return endpointId;
     }
 
+    /** The container cluster this points to */
     public String containerId() {
         return containerId;
     }
 
-    public Set<RegionName> regions() {
-        return regions;
+    /** The regions of this points to */
+    public List<RegionName> regions() {
+        return targets.stream().map(Target::region).collect(Collectors.toUnmodifiableList());
+    }
+
+    /** The level of targets in this */
+    public Level level() {
+        return level;
     }
 
-    public Endpoint withRegions(Set<String> regions) {
-        return new Endpoint(endpointId, containerId, regions);
+    /** The targets this points to */
+    public List<Target> targets() {
+        return targets;
+    }
+
+    /** Returns a copy of this with targets set to given targets */
+    public Endpoint withTargets(List<Target> targets) {
+        return new Endpoint(endpointId, containerId, level, targets);
     }
 
     @Override
@@ -70,20 +103,81 @@ public class Endpoint {
         if (this == o) return true;
         if (o == null || getClass() != o.getClass()) return false;
         Endpoint endpoint = (Endpoint) o;
-        return Objects.equals(endpointId, endpoint.endpointId) &&
-                Objects.equals(containerId, endpoint.containerId) &&
-                Objects.equals(regions, endpoint.regions);
+        return endpointId.equals(endpoint.endpointId) && containerId.equals(endpoint.containerId) && level == endpoint.level && targets.equals(endpoint.targets);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(endpointId, containerId, regions);
+        return Objects.hash(endpointId, containerId, level, targets);
     }
 
     @Override
     public String toString() {
+        if (level == Level.application) {
+            return "endpoint '" + endpointId() + "' (cluster " + containerId + ") -> " +
+                   targets.stream().map(Target::toString).sorted()
+                          .collect(Collectors.joining(", "));
+        }
         return "endpoint '" + endpointId() + "' (cluster " + containerId + ") -> " +
-               regions.stream().map(RegionName::value).sorted().collect(Collectors.joining(", "));
+               targets.stream().map(Target::region).map(RegionName::value).sorted()
+                      .collect(Collectors.joining(", "));
+    }
+
+    /** The level of targets in an endpoint */
+    public enum Level {
+        application,
+        instance,
+    }
+
+    /** A target of an endpoint */
+    public static class Target {
+
+        private final RegionName region;
+        private final InstanceName instance;
+        private final int weight;
+
+        public Target(RegionName region, InstanceName instance, int weight) {
+            this.region = Objects.requireNonNull(region);
+            this.instance = Objects.requireNonNull(instance);
+            this.weight = weight;
+            if (weight < 0 || weight > 100) {
+                throw new IllegalArgumentException("Target must have weight in range [0, 100], got " + weight);
+            }
+        }
+
+        /** The region this points to */
+        public RegionName region() {
+            return region;
+        }
+
+        /** The instance this points to */
+        public InstanceName instance() {
+            return instance;
+        }
+
+        /** The routing weight of this target */
+        public int weight() {
+            return weight;
+        }
+
+        @Override
+        public String toString() {
+            return "region=" + region + ",instance=" + instance + ",weight=" + weight;
+        }
+
+        @Override
+        public boolean equals(Object o) {
+            if (this == o) return true;
+            if (o == null || getClass() != o.getClass()) return false;
+            Target target = (Target) o;
+            return weight == target.weight && region.equals(target.region) && instance.equals(target.instance);
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(region, instance, weight);
+        }
+
     }
 
 }
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
index 56539c2eb84..17a18cbb8c2 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
@@ -31,13 +31,11 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -97,33 +95,38 @@ public class DeploymentSpecXmlReader {
     public DeploymentSpec read(String xmlForm) {
         Element root = XML.getDocument(xmlForm).getDocumentElement();
         if ( ! root.getTagName().equals(deploymentTag))
-            throw new IllegalArgumentException("The root tag must be <deployment>");
+            illegal("The root tag must be <deployment>");
 
         if (isEmptySpec(root))
             return DeploymentSpec.empty;
 
         List<Step> steps = new ArrayList<>();
+        List<Endpoint> applicationEndpoints = List.of();
         if ( ! containsTag(instanceTag, root)) { // deployment spec skipping explicit instance -> "default" instance
             steps.addAll(readInstanceContent("default", root, new MutableOptional<>(), root));
         }
         else {
             if (XML.getChildren(root).stream().anyMatch(child -> child.getTagName().equals(prodTag)))
-                throw new IllegalArgumentException("A deployment spec cannot have both a <prod> tag and an " +
-                                                   "<instance> tag under the root: " +
-                                                   "Wrap the prod tags inside the appropriate instance");
-
-            for (Element topLevelTag : XML.getChildren(root)) {
-                if (topLevelTag.getTagName().equals(instanceTag))
-                    steps.addAll(readInstanceContent(topLevelTag.getAttribute(idAttribute), topLevelTag, new MutableOptional<>(), root));
-                else
-                    steps.addAll(readNonInstanceSteps(topLevelTag, new MutableOptional<>(), root)); // (No global service id here)
+                illegal("A deployment spec cannot have both a <prod> tag and an " +
+                        "<instance> tag under the root: " +
+                        "Wrap the prod tags inside the appropriate instance");
+
+            for (Element child : XML.getChildren(root)) {
+                String tagName = child.getTagName();
+                if (tagName.equals(instanceTag)) {
+                    steps.addAll(readInstanceContent(child.getAttribute(idAttribute), child, new MutableOptional<>(), root));
+                } else {
+                    steps.addAll(readNonInstanceSteps(child, new MutableOptional<>(), root)); // (No global service id here)
+                }
             }
+            applicationEndpoints = readEndpoints(root, Optional.empty(), steps);
         }
 
         return new DeploymentSpec(steps,
                                   optionalIntegerAttribute(majorVersionTag, root),
                                   stringAttribute(athenzDomainAttribute, root).map(AthenzDomain::from),
                                   stringAttribute(athenzServiceAttribute, root).map(AthenzService::from),
+                                  applicationEndpoints,
                                   xmlForm);
     }
 
@@ -140,7 +143,7 @@ public class DeploymentSpecXmlReader {
                                                              MutableOptional<String> globalServiceId,
                                                              Element parentTag) {
         if (instanceNameString.isBlank())
-            throw new IllegalArgumentException("<instance> attribute 'id' must be specified, and not be blank");
+            illegal("<instance> attribute 'id' must be specified, and not be blank");
 
         // If this is an absolutely empty instance, or the implicit "default" instance but without content, ignore it
         if (XML.getChildren(instanceTag).isEmpty() && (instanceTag.getAttributes().getLength() == 0 || instanceTag == parentTag))
@@ -160,7 +163,7 @@ public class DeploymentSpecXmlReader {
         List<Step> steps = new ArrayList<>();
         for (Element instanceChild : XML.getChildren(instanceTag))
             steps.addAll(readNonInstanceSteps(instanceChild, globalServiceId, instanceChild));
-        List<Endpoint> endpoints = readEndpoints(instanceTag);
+        List<Endpoint> endpoints = readEndpoints(instanceTag, Optional.of(instanceNameString), steps);
 
         // Build and return instances with these values
         return Arrays.stream(instanceNameString.split(","))
@@ -194,7 +197,7 @@ public class DeploymentSpecXmlReader {
         if (prodTag.equals(stepTag.getTagName()))
             globalServiceId.set(readGlobalServiceId(stepTag));
         else if (readGlobalServiceId(stepTag).isPresent())
-            throw new IllegalArgumentException("Attribute 'global-service-id' is only valid on 'prod' tag.");
+            illegal("Attribute 'global-service-id' is only valid on 'prod' tag.");
 
         switch (stepTag.getTagName()) {
             case testTag:
@@ -254,7 +257,7 @@ public class DeploymentSpecXmlReader {
             Optional<Role> roleAttribute = stringAttribute("role", emailElement).map(Role::fromValue);
             When when = stringAttribute("when", emailElement).map(When::fromValue).orElse(defaultWhen);
             if (addressAttribute.isPresent() == roleAttribute.isPresent())
-                throw new IllegalArgumentException("Exactly one of 'role' and 'address' must be present in 'email' elements.");
+                illegal("Exactly one of 'role' and 'address' must be present in 'email' elements.");
 
             addressAttribute.ifPresent(address -> emailAddresses.get(when).add(address));
             roleAttribute.ifPresent(role -> emailRoles.get(when).add(role));
@@ -262,36 +265,62 @@ public class DeploymentSpecXmlReader {
         return Notifications.of(emailAddresses, emailRoles);
     }
 
-    private List<Endpoint> readEndpoints(Element parent) {
+    private List<Endpoint> readEndpoints(Element parent, Optional<String> instance, List<Step> steps) {
         var endpointsElement = XML.getChild(parent, endpointsTag);
-        if (endpointsElement == null)
-            return List.of();
-
-        var endpoints = new LinkedHashMap<String, Endpoint>();
+        if (endpointsElement == null) return List.of();
 
+        Endpoint.Level level = instance.isEmpty() ? Endpoint.Level.application : Endpoint.Level.instance;
+        Map<String, Endpoint> endpoints = new LinkedHashMap<>();
         for (var endpointElement : XML.getChildren(endpointsElement, endpointTag)) {
             String endpointId = stringAttribute("id", endpointElement).orElse(Endpoint.DEFAULT_ID);
             String containerId = requireStringAttribute("container-id", endpointElement);
-            Set<String> regions = new HashSet<>();
+            String msgPrefix = (level == Endpoint.Level.application ? "Application-level" : "Instance-level") +
+                               " endpoint '" + endpointId + "': ";
 
+            List<Endpoint.Target> targets = new ArrayList<>();
             for (var regionElement : XML.getChildren(endpointElement, "region")) {
-                var region = regionElement.getTextContent();
-                if (region == null || region.isBlank()) {
-                    throw new IllegalArgumentException("Empty 'region' element in 'endpoint' tag.");
+                String region = regionElement.getTextContent();
+                if (region == null || region.isBlank()) illegal(msgPrefix + "empty 'region' element");
+                Optional<String> instanceFromAttribute = stringAttribute("instance", regionElement);
+                Optional<String> weightFromAttribute = stringAttribute("weight", regionElement);
+                if (level == Endpoint.Level.application) {
+                    if (instanceFromAttribute.isEmpty()) illegal(msgPrefix + "element 'region' must have 'instance' attribute");
+                    if (weightFromAttribute.isEmpty()) illegal(msgPrefix + "element 'region' must have 'weight' attribute");
+                } else {
+                    if (instanceFromAttribute.isPresent()) illegal(msgPrefix + "element 'region' cannot have 'instance' attribute");
+                    if (weightFromAttribute.isPresent()) illegal(msgPrefix + "element 'region' cannot have 'weight' attribute");
+                    instanceFromAttribute = instance;
                 }
-                if (regions.contains(region)) {
-                    throw new IllegalArgumentException("Duplicate 'region' element in 'endpoint' tag: " + region);
+                int weight = 1;
+                if (weightFromAttribute.isPresent()) {
+                    try {
+                        weight = Integer.parseInt(weightFromAttribute.get());
+                    } catch (NumberFormatException e) {
+                        throw new IllegalArgumentException(msgPrefix + "invalid weight value '" + weightFromAttribute.get() + "'");
+                    }
                 }
-                regions.add(region);
+                targets.add(new Endpoint.Target(RegionName.from(region),
+                                                InstanceName.from(instanceFromAttribute.get()),
+                                                weight));
+            }
+            if (targets.isEmpty() && level == Endpoint.Level.instance) {
+                // No explicit targets given for instance level endpoint. Include all declared regions by default
+                InstanceName instanceName = instance.map(InstanceName::from).get();
+                steps.stream()
+                     .filter(step -> step.concerns(Environment.prod))
+                     .flatMap(step -> step.zones().stream())
+                     .flatMap(zone -> zone.region().stream())
+                     .distinct()
+                     .map(region -> new Endpoint.Target(region, instanceName, 1))
+                     .forEach(targets::add);
             }
 
-            Endpoint endpoint = new Endpoint(endpointId, containerId, regions);
+            Endpoint endpoint = new Endpoint(endpointId, containerId, level, targets);
             if (endpoints.containsKey(endpoint.endpointId())) {
-                throw new IllegalArgumentException("Duplicate attribute 'id' on 'endpoint': " + endpoint.endpointId());
+                illegal("Endpoint ID '" + endpoint.endpointId() + "' is specified multiple times");
             }
             endpoints.put(endpoint.endpointId(), endpoint);
         }
-
         return List.copyOf(endpoints.values());
     }
 
@@ -303,9 +332,9 @@ public class DeploymentSpecXmlReader {
         for (int i = 0; i < tags.size(); i++) {
             if (tags.get(i).equals(blockChangeTag)) {
                 String constraint = "<block-change> must be placed after <test> and <staging> and before <prod>";
-                if (containsAfter(i, testTag, tags)) throw new IllegalArgumentException(constraint);
-                if (containsAfter(i, stagingTag, tags)) throw new IllegalArgumentException(constraint);
-                if (containsBefore(i, prodTag, tags)) throw new IllegalArgumentException(constraint);
+                if (containsAfter(i, testTag, tags)) illegal(constraint);
+                if (containsAfter(i, stagingTag, tags)) illegal(constraint);
+                if (containsBefore(i, prodTag, tags)) illegal(constraint);
             }
         }
     }
@@ -464,6 +493,10 @@ public class DeploymentSpecXmlReader {
                      .findFirst();
     }
 
+    private static void illegal(String message) {
+        throw new IllegalArgumentException(message);
+    }
+
     private static class MutableOptional<T> {
 
         private Optional<T> value = Optional.empty();
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
index 7cc209f9f7f..4961007e246 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecTest.java
@@ -3,6 +3,7 @@ package com.yahoo.config.application.api;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.provision.Environment;
+import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 import org.junit.Test;
 
@@ -1130,7 +1131,7 @@ public class DeploymentSpecTest {
                 spec.requireInstance("default").endpoints().stream().map(Endpoint::containerId).collect(Collectors.toList())
         );
 
-        assertEquals(Set.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions());
+        assertEquals(List.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions());
     }
 
     @Test
@@ -1179,6 +1180,103 @@ public class DeploymentSpecTest {
         assertEquals(Set.of("us-east", "us-west"), endpointRegions("default", spec));
     }
 
+    @Test
+    public void instanceEndpointDisallowsApplicationLevelAttributes() {
+        String xmlForm = "<deployment>\n" +
+                         "  <prod>\n" +
+                         "    <region active=\"true\">us-east</region>\n" +
+                         "    <region active=\"true\">us-west</region>\n" +
+                         "  </prod>\n" +
+                         "  <endpoints>\n" +
+                         "    <endpoint id=\"foo\" container-id=\"bar\">\n" +
+                         "      <region %s>us-east</region>\n" +
+                         "    </endpoint>\n" +
+                         "  </endpoints>\n" +
+                         "</deployment>";
+        assertInvalid(String.format(xmlForm, "instance='foo'"),
+                      "Instance-level endpoint 'foo': element 'region' cannot have 'instance' attribute");
+        assertInvalid(String.format(xmlForm, "weight='1'"),
+                      "Instance-level endpoint 'foo': element 'region' cannot have 'weight' attribute");
+    }
+
+    @Test
+    public void applicationLevelEndpointRequiresAttributes() {
+        String xmlForm = "<deployment>\n" +
+                         "  <instance id=\"beta\">\n" +
+                         "    <prod>\n" +
+                         "      <region active='true'>us-west-1</region>\n" +
+                         "      <region active='true'>us-east-3</region>\n" +
+                         "    </prod>\n" +
+                         "  </instance>\n" +
+                         "  <instance id=\"main\">\n" +
+                         "    <prod>\n" +
+                         "      <region active='true'>us-west-1</region>\n" +
+                         "      <region active='true'>us-east-3</region>\n" +
+                         "    </prod>\n" +
+                         "  </instance>\n" +
+                         "  <endpoints>\n" +
+                         "    <endpoint id=\"foo\" container-id=\"qrs\">\n" +
+                         "      <region %s>%s</region>\n" +
+                         "    </endpoint>\n" +
+                         "  </endpoints>\n" +
+                         "</deployment>\n";
+        assertInvalid(String.format(xmlForm, "", "us-west-1"), "Application-level endpoint 'foo': element 'region' must have 'instance' attribute");
+        assertInvalid(String.format(xmlForm, "instance='beta'", "us-west-1"), "Application-level endpoint 'foo': element 'region' must have 'weight' attribute");
+        assertInvalid(String.format(xmlForm, "instance='foo' weight='1'", "us-west-1"), "Application-level endpoint 'foo': targets undeclared instance 'foo'");
+        assertInvalid(String.format(xmlForm, "instance='beta' weight='foo'", "us-west-1"), "Application-level endpoint 'foo': invalid weight value 'foo'");
+        assertInvalid(String.format(xmlForm, "instance='beta' weight='1'", "eu-north-1"), "Application-level endpoint 'foo': targets undeclared region 'eu-north-1' in instance 'beta'");
+    }
+
+    @Test
+    public void applicationLevelEndpoint() {
+        DeploymentSpec spec = DeploymentSpec.fromXml("<deployment>\n" +
+                                                     "  <instance id=\"beta\">\n" +
+                                                     "    <prod>\n" +
+                                                     "      <region active='true'>us-west-1</region>\n" +
+                                                     "      <region active='true'>us-east-3</region>\n" +
+                                                     "    </prod>\n" +
+                                                     "  </instance>\n" +
+                                                     "  <instance id=\"main\">\n" +
+                                                     "    <prod>\n" +
+                                                     "      <region active='true'>us-west-1</region>\n" +
+                                                     "      <region active='true'>us-east-3</region>\n" +
+                                                     "    </prod>\n" +
+                                                     "    <endpoints>\n" +
+                                                     "      <endpoint id=\"glob\" container-id=\"music\"/>\n" +
+                                                     "    </endpoints>\n" +
+                                                     "  </instance>\n" +
+                                                     "  <endpoints>\n" +
+                                                     "    <endpoint id=\"foo\" container-id=\"movies\">\n" +
+                                                     "      <region instance=\"beta\" weight=\"2\">us-west-1</region>\n" +
+                                                     "      <region instance=\"main\" weight=\"8\">us-west-1</region>\n" +
+                                                     "    </endpoint>\n" +
+                                                     "    <endpoint id=\"bar\" container-id=\"music\">\n" +
+                                                     "      <region instance=\"main\" weight=\"10\">us-east-3</region>\n" +
+                                                     "    </endpoint>\n" +
+                                                     "  </endpoints>\n" +
+                                                     "</deployment>\n");
+        assertEquals(List.of(new Endpoint("foo", "movies", Endpoint.Level.application,
+                                          List.of(new Endpoint.Target(RegionName.from("us-west-1"), InstanceName.from("beta"), 2),
+                                                  new Endpoint.Target(RegionName.from("us-west-1"), InstanceName.from("main"), 8))),
+                             new Endpoint("bar", "music", Endpoint.Level.application,
+                                          List.of(new Endpoint.Target(RegionName.from("us-east-3"), InstanceName.from("main"), 10)))),
+                             spec.endpoints());
+        assertEquals(List.of(new Endpoint("glob", "music", Endpoint.Level.instance,
+                                          List.of(new Endpoint.Target(RegionName.from("us-west-1"), InstanceName.from("main"), 1),
+                                                  new Endpoint.Target(RegionName.from("us-east-3"), InstanceName.from("main"), 1)))),
+                     spec.requireInstance("main").endpoints());
+    }
+
+    private static void assertInvalid(String deploymentSpec, String errorMessagePart) {
+        try {
+            DeploymentSpec.fromXml(deploymentSpec);
+            fail("Expected exception");
+        } catch (IllegalArgumentException e) {
+            assertTrue("\"" + e.getMessage() + "\" contains \"" + errorMessagePart + "\"",
+                       e.getMessage().contains(errorMessagePart));
+        }
+    }
+
     private static void assertInvalidEndpoints(String endpointsBody) {
         try {
             endpointIds(endpointsBody);
diff --git a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
index 8a1af9e7a8f..efed8ecc06c 100644
--- a/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
+++ b/config-model-api/src/test/java/com/yahoo/config/application/api/DeploymentSpecWithoutInstanceTest.java
@@ -3,7 +3,6 @@ package com.yahoo.config.application.api;
 
 import com.google.common.collect.ImmutableSet;
 import com.yahoo.config.provision.Environment;
-import com.yahoo.config.provision.InstanceName;
 import com.yahoo.config.provision.RegionName;
 import org.junit.Test;
 
@@ -678,7 +677,7 @@ public class DeploymentSpecWithoutInstanceTest {
                 spec.requireInstance("default").endpoints().stream().map(Endpoint::containerId).collect(Collectors.toList())
         );
 
-        assertEquals(Set.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions());
+        assertEquals(List.of(RegionName.from("us-east")), spec.requireInstance("default").endpoints().get(0).regions());
     }
 
     @Test
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
index 302ac1d393d..9ab5096ec8f 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/pkg/ApplicationPackageValidator.java
@@ -9,7 +9,6 @@ import com.yahoo.config.application.api.ValidationOverrides;
 import com.yahoo.config.provision.CloudName;
 import com.yahoo.config.provision.Environment;
 import com.yahoo.config.provision.InstanceName;
-import com.yahoo.config.provision.RegionName;
 import com.yahoo.config.provision.zone.ZoneApi;
 import com.yahoo.config.provision.zone.ZoneId;
 import com.yahoo.vespa.hosted.controller.Application;
@@ -172,15 +171,16 @@ public class ApplicationPackageValidator {
         return endpoints;
     }
 
-    /** Returns global service ID as a endpoint, if any global service ID is set */
+    /** Returns global service ID as an endpoint, if any global service ID is set */
     private static Optional<Endpoint> legacyEndpoint(DeploymentInstanceSpec instance) {
         return instance.globalServiceId().map(globalServiceId -> {
-            var regions = instance.zones().stream()
+            var targets = instance.zones().stream()
                                   .filter(zone -> zone.environment().isProduction())
                                   .flatMap(zone -> zone.region().stream())
-                                  .map(RegionName::value)
-                                  .collect(Collectors.toSet());
-            return new Endpoint(Optional.of(EndpointId.defaultId().id()), globalServiceId, regions);
+                                  .distinct()
+                                  .map(region -> new Endpoint.Target(region, instance.name(), 1))
+                                  .collect(Collectors.toList());
+            return new Endpoint(EndpointId.defaultId().id(), globalServiceId, Endpoint.Level.instance, targets);
         });
     }
 
diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
index e15c8fa0ac4..5b24f39717b 100644
--- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
+++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/rotation/RotationRepository.java
@@ -22,6 +22,7 @@ import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
+import java.util.Set;
 import java.util.function.Function;
 import java.util.logging.Logger;
 import java.util.stream.Collectors;
@@ -142,7 +143,7 @@ public class RotationRepository {
             } else { // Rotation already assigned to this endpoint, reuse it
                 rotationId = assignedRotation.rotationId();
             }
-            assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, endpoint.regions()));
+            assignments.add(new AssignedRotation(ClusterSpec.Id.from(endpoint.containerId()), endpointId, rotationId, Set.copyOf(endpoint.regions())));
         }
         return Collections.unmodifiableList(assignments);
     }