Support configuring BCP structure

4 files changed, 221 insertions, 14 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java
new file mode 100644
index 00000000000..af369dc2672
--- /dev/null
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/Bcp.java
@@ -0,0 +1,122 @@
+package com.yahoo.config.application.api;
+
+import com.yahoo.config.provision.RegionName;
+
+import java.time.Duration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Defines the BCP structure for an instance in a deployment spec:
+ * A list of region groups where each group contains a set of regions
+ * which will handle the traffic of a member in the group when it becomes unreachable.
+ *
+ * This is used to make bcp-aware autoscaling decisions. If no explicit BCP spec
+ * is provided, it is assumed that a regions traffic will be divided equally over all
+ * the other regions when it becomes unreachable - i.e a single BCP group is implicitly
+ * defined having all defined production regions as members with fraction 1.0.
+ *
+ * It is assumed that the traffic of the unreachable region is distributed
+ * evenly to the other members of the group.
+ *
+ * A region can be a fractional member of a group, in which case it is assumed that
+ * region will only handle that fraction of its share of the unreachable regions traffic,
+ * and symmetrically that the other members of the group will only handle that fraction
+ * of the fraction regions traffic if it becomes unreachable.
+ *
+ * Each production region defined in the instance must have fractional memberships in groups that sums to exactly one.
+ *
+ * If a group has one member it will not set aside any capacity for BCP.
+ * If a group has more than two members, the system will attempt to provision capacity
+ * for BCP also when a region is unreachable. That is, if there are three member regions, A, B and C,
+ * each handling 100 qps, then they each aim to handle 150 in case one goes down. If C goes down,
+ * A and B will now handle 150 each, but will each aim to handle 300 each in case the other goes down.
+ *
+ * @author bratseth
+ */
+public class Bcp {
+
+    private static final Bcp empty = new Bcp(List.of());
+
+    private final List<Group> groups;
+
+    public Bcp(List<Group> groups) {
+        totalMembershipSumsToOne(groups);
+        this.groups = List.copyOf(groups);
+    }
+
+    public List<Group> groups() { return groups; }
+
+    /** Returns the set of regions declared in the groups of this. */
+    public Set<RegionName> regions() {
+        return groups.stream().flatMap(group -> group.members().stream()).map(member -> member.region()).collect(Collectors.toSet());
+    }
+
+    public boolean isEmpty() { return groups.isEmpty(); }
+
+    /** Returns this bcp spec, or if it is empty, the given bcp spec. */
+    public Bcp orElse(Bcp other) {
+        return this.isEmpty() ? other : this;
+    }
+
+    private void totalMembershipSumsToOne(List<Group> groups) {
+        Map<RegionName, Double> totalMembership = new HashMap<>();
+        for (var group : groups) {
+            for (var member : group.members())
+                totalMembership.compute(member.region(), (__, fraction) -> fraction == null ? member.fraction()
+                                                                                            : fraction + member.fraction());
+        }
+        for (var entry : totalMembership.entrySet()) {
+            if (entry.getValue() != 1.0)
+                throw new IllegalArgumentException("Illegal BCP spec: All regions must have total membership fractions summing to 1.0, but " +
+                                                   entry.getKey() + " sums to " + entry.getValue());
+        }
+    }
+
+    public static Bcp empty() { return empty; }
+
+    @Override
+    public String toString() {
+        if (isEmpty()) return "empty BCP";
+        return "BCP of " + groups;
+    }
+
+    public static class Group {
+
+        private final Duration deadline;
+        private final List<RegionMember> members;
+
+        public Group(List<RegionMember> members, Duration deadline) {
+            this.members = List.copyOf(members);
+            this.deadline = deadline;
+        }
+
+        public List<RegionMember> members() { return members; }
+
+        /**
+         * Returns the max time until the other regions must be able to handle the additional traffic
+         * when a region becomes unreachable, which by default is Duration.ZERO.
+         */
+        public Duration deadline() { return deadline; }
+
+        @Override
+        public String toString() {
+            return "BCP group of " + members;
+        }
+
+    }
+
+    public record RegionMember(RegionName region, double fraction) {
+
+        public RegionMember {
+            if (fraction < 0 || fraction > 1)
+                throw new IllegalArgumentException("Fraction must be a number between 0.0 and 1.0, but got " + fraction);
+        }
+
+
+    }
+
+}
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
index b36c1409459..4b30734365d 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentInstanceSpec.java
@@ -61,6 +61,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
     private final Notifications notifications;
     private final List<Endpoint> endpoints;
     private final Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints;
+    private final Bcp bcp;
 
     public DeploymentInstanceSpec(InstanceName name,
                                   Tags tags,
@@ -77,6 +78,7 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
                                   Notifications notifications,
                                   List<Endpoint> endpoints,
                                   Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpoints,
+                                  Bcp bcp,
                                   Instant now) {
         super(steps);
         this.name = Objects.requireNonNull(name);
@@ -101,8 +103,9 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
         Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> zoneEndpointsCopy =  new HashMap<>();
         for (var entry : zoneEndpoints.entrySet()) zoneEndpointsCopy.put(entry.getKey(), Collections.unmodifiableMap(new HashMap<>(entry.getValue())));
         this.zoneEndpoints = Collections.unmodifiableMap(zoneEndpointsCopy);
+        this.bcp = Objects.requireNonNull(bcp);
         validateZones(new HashSet<>(), new HashSet<>(), this);
-        validateEndpoints(steps(), globalServiceId, this.endpoints);
+        validateEndpoints(globalServiceId, this.endpoints);
         validateChangeBlockers(changeBlockers, now);
     }
 
@@ -144,25 +147,41 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
     }
 
     /** Throw an IllegalArgumentException if an endpoint refers to a region that is not declared in 'prod' */
-    private void validateEndpoints(List<DeploymentSpec.Step> steps, Optional<String> globalServiceId, List<Endpoint> endpoints) {
+    private void validateEndpoints(Optional<String> globalServiceId, List<Endpoint> endpoints) {
         if (globalServiceId.isPresent() && ! endpoints.isEmpty()) {
             throw new IllegalArgumentException("Providing both 'endpoints' and 'global-service-id'. Use only 'endpoints'.");
         }
 
-        var stepZones = steps.stream()
-                             .flatMap(s -> s.zones().stream())
-                             .flatMap(z -> z.region().stream())
-                             .collect(Collectors.toSet());
-
+        var regions = prodRegions();
         for (var endpoint : endpoints){
             for (var endpointRegion : endpoint.regions()) {
-                if (! stepZones.contains(endpointRegion)) {
+                if (! regions.contains(endpointRegion)) {
                     throw new IllegalArgumentException("Region used in endpoint that is not declared in 'prod': " + endpointRegion);
                 }
             }
         }
     }
 
+    /** Validates the given BCP instance (which is owned by this, or if none, a default) against this instance. */
+    void validateBcp(Bcp bcp) {
+        if (bcp.isEmpty()) return;
+        if ( ! prodRegions().equals(bcp.regions()))
+            throw new IllegalArgumentException("BCP and deployment mismatch in " + this + ": " +
+                                               "A <bcp> element must place all deployed production regions in " +
+                                               "at least one group, and declare no extra regions. " +
+                                               "Deployed regions: " + prodRegions() +
+                                               ". BCP regions: " + bcp.regions());
+}
+    /** Returns the production regions the steps of this specifies a deployment to. */
+    private Set<RegionName> prodRegions() {
+        return steps().stream()
+                      .flatMap(s -> s.zones().stream())
+                      .filter(zone -> zone.environment().isProduction())
+                      .flatMap(z -> z.region().stream())
+                      .collect(Collectors.toSet());
+     }
+
+
     private void validateChangeBlockers(List<DeploymentSpec.ChangeBlocker> changeBlockers, Instant now) {
         // Find all possible dates an upgrade block window can start
         Stream<Instant> blockingFrom = changeBlockers.stream()
@@ -256,6 +275,9 @@ public class DeploymentInstanceSpec extends DeploymentSpec.Steps {
     /** Returns the rotations configuration of these instances */
     public List<Endpoint> endpoints() { return endpoints; }
 
+    /** Returns the BCP spec declared in this specified instance, or BcpSpec.empty() if none. */
+    public Bcp bcp() { return bcp; }
+
     /** Returns whether this instance deploys to the given zone, either implicitly or explicitly */
     public boolean deploysTo(Environment environment, RegionName region) {
         return zones().stream().anyMatch(zone -> zone.concerns(environment, Optional.of(region)));
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
index cbdb5bd6bcc..0e9841fe5be 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/DeploymentSpec.java
@@ -18,9 +18,7 @@ import java.time.Duration;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Comparator;
-import java.util.HashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.stream.Collectors;
@@ -47,6 +45,7 @@ public class DeploymentSpec {
                                                                   Optional.empty(),
                                                                   Optional.empty(),
                                                                   List.of(),
+                                                                  Bcp.empty(),
                                                                   "<deployment version='1.0'/>",
                                                                   List.of());
 
@@ -58,6 +57,7 @@ public class DeploymentSpec {
     private final Optional<AthenzService> athenzService;
     private final Optional<CloudAccount> cloudAccount;
     private final List<Endpoint> endpoints;
+    private final Bcp bcp;
     private final List<DeprecatedElement> deprecatedElements;
 
     private final String xmlForm;
@@ -68,6 +68,7 @@ public class DeploymentSpec {
                           Optional<AthenzService> athenzService,
                           Optional<CloudAccount> cloudAccount,
                           List<Endpoint> endpoints,
+                          Bcp bcp,
                           String xmlForm,
                           List<DeprecatedElement> deprecatedElements) {
         this.steps = List.copyOf(Objects.requireNonNull(steps));
@@ -77,11 +78,13 @@ public class DeploymentSpec {
         this.cloudAccount = Objects.requireNonNull(cloudAccount);
         this.xmlForm = Objects.requireNonNull(xmlForm);
         this.endpoints = List.copyOf(Objects.requireNonNull(endpoints));
+        this.bcp = Objects.requireNonNull(bcp);
         this.deprecatedElements = List.copyOf(Objects.requireNonNull(deprecatedElements));
         validateTotalDelay(steps);
         validateUpgradePoliciesOfIncreasingConservativeness(steps);
         validateAthenz();
         validateApplicationEndpoints();
+        validateBcp();
     }
 
     /** Throw an IllegalArgumentException if the total delay exceeds 24 hours */
@@ -158,13 +161,16 @@ public class DeploymentSpec {
         }
     }
 
+    private void validateBcp() {
+        for (var instance : instances())
+            instance.validateBcp(instance.bcp().orElse(bcp()));
+    }
+
     /** Returns the major version this application is pinned to, or empty (default) to allow all major versions */
     public Optional<Integer> majorVersion() { return majorVersion; }
 
     /** Returns the deployment steps of this in the order they will be performed */
-    public List<Step> steps() {
-        return steps;
-    }
+    public List<Step> steps() { return steps; }
 
     /** Returns the Athenz domain set on the root tag, if any */
     public Optional<AthenzDomain> athenzDomain() { return athenzDomain; }
@@ -203,6 +209,9 @@ public class DeploymentSpec {
                                  .orElse(ZoneEndpoint.defaultEndpoint);
     }
 
+    /** Returns the default BCP spec for instances, or Bcp.empty() if none are defined. */
+    public Bcp bcp() { return bcp; }
+
     /** Returns the XML form of this spec, or null if it was not created by fromXml, nor is empty */
     public String xmlForm() { return xmlForm; }
 
diff --git a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
index fb6d834f783..be6be5566a8 100644
--- a/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
+++ b/config-model-api/src/main/java/com/yahoo/config/application/api/xml/DeploymentSpecXmlReader.java
@@ -1,6 +1,7 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.config.application.api.xml;
 
+import com.yahoo.config.application.api.Bcp;
 import com.yahoo.config.application.api.DeploymentInstanceSpec;
 import com.yahoo.config.application.api.DeploymentSpec;
 import com.yahoo.config.application.api.DeploymentSpec.DeclaredTest;
@@ -46,7 +47,6 @@ import java.time.Instant;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
-import java.util.Comparator;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
@@ -54,6 +54,7 @@ import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.OptionalDouble;
 import java.util.Set;
 import java.util.function.Function;
 import java.util.stream.Stream;
@@ -161,6 +162,7 @@ public class DeploymentSpecXmlReader {
                                   stringAttribute(athenzServiceAttribute, root).map(AthenzService::from),
                                   stringAttribute(cloudAccountAttribute, root).map(CloudAccount::from),
                                   applicationEndpoints,
+                                  readBcp(root),
                                   xmlForm,
                                   deprecatedElements);
     }
@@ -228,6 +230,7 @@ public class DeploymentSpecXmlReader {
                                                              notifications,
                                                              endpoints,
                                                              zoneEndpoints,
+                                                             readBcp(instanceElement),
                                                              now))
                      .toList();
     }
@@ -455,6 +458,24 @@ public class DeploymentSpecXmlReader {
         validateAndConsolidate(endpointsByZone, zoneEndpoints);
     }
 
+    static Bcp readBcp(Element element) {
+        Element bcpElement = XML.getChild(element, "bcp");
+        if (bcpElement == null) return Bcp.empty();
+
+        List<Bcp.Group> groups = new ArrayList<>();
+        for (Element groupElement : XML.getChildren(bcpElement, "group")) {
+            List<Bcp.RegionMember> regions = new ArrayList<>();
+            for (Element regionElement : XML.getChildren(groupElement, "region")) {
+                RegionName region = RegionName.from(XML.getValue(regionElement));
+                double fraction = toDouble(XML.attribute("fraction", regionElement).orElse(null), "fraction").orElse(1.0);
+                regions.add(new Bcp.RegionMember(region, fraction));
+            }
+            Duration deadline = XML.attribute("deadline", groupElement).map(value -> toDuration(value, "deadline")).orElse(Duration.ZERO);
+            groups.add(new Bcp.Group(regions, deadline));
+        }
+        return new Bcp(groups);
+    }
+
     static void validateAndConsolidate(Map<String, Map<RegionName, List<ZoneEndpoint>>> in, Map<ClusterSpec.Id, Map<ZoneId, ZoneEndpoint>> out) {
         in.forEach((cluster, regions) -> {
             List<ZoneEndpoint> wildcards = regions.remove(null);
@@ -713,6 +734,39 @@ public class DeploymentSpecXmlReader {
                      .findFirst();
     }
 
+    /**
+     * Returns a string consisting of a number followed by "m" or "M" to a duration of that number of minutes,
+     * or zero duration if null of blank.
+     */
+    private static Duration toDuration(String minutesSpec, String sourceDescription) {
+        try {
+            if (minutesSpec == null || minutesSpec.isBlank()) return Duration.ZERO;
+            minutesSpec = minutesSpec.trim().toLowerCase();
+            if ( ! minutesSpec.endsWith("m"))
+                throw new IllegalArgumentException("Must end by 'm'");
+            try {
+                return Duration.ofMinutes(Integer.parseInt(minutesSpec.substring(0, minutesSpec.length() - 1)));
+            }
+            catch (NumberFormatException e) {
+                throw new IllegalArgumentException("Must be an integer number of minutes followed by 'm'");
+            }
+        }
+        catch (IllegalArgumentException e) {
+            throw new IllegalArgumentException("Illegal " + sourceDescription + " '" + minutesSpec + "'", e);
+        }
+    }
+
+    private static OptionalDouble toDouble(String value, String sourceDescription) {
+        try {
+            if (value == null || value.isBlank()) return OptionalDouble.empty();
+            return OptionalDouble.of(Double.parseDouble(value));
+        }
+        catch (NumberFormatException e) {
+            throw new IllegalArgumentException("Illegal " + sourceDescription + " '" + value + "': " +
+                                               "Must be a number between 0.0 and 1.0");
+        }
+    }
+
     private static void illegal(String message) {
         throw new IllegalArgumentException(message);
     }