Add support for max groups allowed down in services.xml

Undocumented for now, need an architecture review on naming and
placement in services.xml

3 files changed, 36 insertions, 7 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index c72aa23a836..540905bef4c 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -83,6 +83,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private boolean useRestrictedDataPlaneBindings = false;
     private Optional<CloudAccount> cloudAccount = Optional.empty();
     private boolean allowUserFilters = true;
+    private boolean allowMoreThanOneContentGroupDown = false;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -140,6 +141,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public Optional<CloudAccount> cloudAccount() { return cloudAccount; }
     @Override public boolean allowUserFilters() { return allowUserFilters; }
     @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only
+    @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown; }
 
     public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) {
         this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim;
@@ -368,6 +370,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setAllowMoreThanOneContentGroupDown(boolean allowMoreThanOneContentGroupDown) {
+        this.allowMoreThanOneContentGroupDown = allowMoreThanOneContentGroupDown;
+        return this;
+    }
+
     public TestProperties setAllowUserFilters(boolean b) { this.allowUserFilters = b; return this; }
 
     public static class Spec implements ConfigServerSpec {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
index 49d63cbfba3..5a96e33c522 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
@@ -94,12 +94,14 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
         tuning.minStorageUpRatio.ifPresent(builder::min_storage_up_ratio);
         tuning.minSplitBits.ifPresent(builder::ideal_distribution_bits);
         tuning.minNodeRatioPerGroup.ifPresent(builder::min_node_ratio_per_group);
+        tuning.maxGroupsAllowedDown.ifPresent(max -> builder.max_number_of_groups_allowed_to_be_down(allowMoreThanOneContentGroupDown ? max : -1));
 
         resourceLimits.getConfig(builder);
-        builder.max_number_of_groups_allowed_to_be_down(allowMoreThanOneContentGroupDown ? 1 : -1);
     }
 
-    private static class ClusterControllerTuning {
+    public ClusterControllerTuning tuning() { return tuning; }
+
+    public static class ClusterControllerTuning {
 
         private final Optional<Double> minNodeRatioPerGroup;
         private final Optional<Duration> initProgressTime;
@@ -109,6 +111,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
         private final Optional<Double> minDistributorUpRatio;
         private final Optional<Double> minStorageUpRatio;
         private final Optional<Integer> minSplitBits;
+        final Optional<Integer> maxGroupsAllowedDown;
 
         ClusterControllerTuning(ModelElement tuning,
                                 Optional<Double> minNodeRatioPerGroup,
@@ -122,6 +125,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                 this.stableStateTimePeriod = Optional.empty();
                 this.minDistributorUpRatio = Optional.empty();
                 this.minStorageUpRatio = Optional.empty();
+                this.maxGroupsAllowedDown = Optional.empty();
             } else {
                 this.initProgressTime = Optional.ofNullable(tuning.childAsDuration("init-progress-time"));
                 this.transitionTime = Optional.ofNullable(tuning.childAsDuration("transition-time"));
@@ -129,8 +133,11 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                 this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period"));
                 this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio"));
                 this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio"));
+                this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down"));
             }
         }
+
+        public Optional<Integer> maxGroupsAllowedDown() { return maxGroupsAllowedDown; }
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 217c26516a9..f1f210b013c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -125,11 +125,6 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
                                                                    deployState.featureFlags().resourceLimitDisk(),
                                                                    deployState.featureFlags().resourceLimitMemory())
                     .build(contentElement);
-            c.clusterControllerConfig = new ClusterControllerConfig.Builder(clusterId,
-                                                                            contentElement,
-                                                                            resourceLimits.getClusterControllerLimits(),
-                                                                            deployState.featureFlags().allowMoreThanOneContentGroupDown(new ClusterSpec.Id(clusterId)))
-                    .build(deployState, c, contentElement.getXml());
             c.search = new ContentSearchCluster.Builder(documentDefinitions,
                                                         globallyDistributedDocuments,
                                                         fractionOfMemoryReserved(clusterId, containers),
@@ -139,6 +134,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
             c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement);
             c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement);
             c.rootGroup = new StorageGroup.Builder(contentElement, context).buildRootGroup(deployState, redundancyBuilder, c);
+            c.clusterControllerConfig = createClusterControllerConfig(contentElement, deployState, c, resourceLimits);
             validateThatGroupSiblingsAreUnique(c.clusterId, c.rootGroup);
             c.search.handleRedundancy(c.redundancy);
             setupSearchCluster(c.search, contentElement, deployState.getDeployLogger());
@@ -164,6 +160,24 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
             return c;
         }
 
+        private ClusterControllerConfig createClusterControllerConfig(ModelElement contentElement,
+                                                                      DeployState deployState,
+                                                                      ContentCluster c,
+                                                                      ClusterResourceLimits resourceLimits) {
+            var config = new ClusterControllerConfig.Builder(c.clusterId,
+                                                             contentElement,
+                                                             resourceLimits.getClusterControllerLimits(),
+                                                             deployState.featureFlags()
+                                                                        .allowMoreThanOneContentGroupDown(new ClusterSpec.Id(c.clusterId)))
+                    .build(deployState, c, contentElement.getXml());
+            config.tuning().maxGroupsAllowedDown().ifPresent(m -> {
+                int numberOfLeafGroups = c.getRootGroup().getNumberOfLeafGroups();
+                if (m > numberOfLeafGroups)
+                    throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + m + ") larger than number of groups (" + numberOfLeafGroups + ")");
+            });
+            return config;
+        }
+
         private void setupSearchCluster(ContentSearchCluster csc, ModelElement element, DeployLogger logger) {
             ContentSearch search = DomContentSearchBuilder.build(element);
             Double visibilityDelay = search.getVisibilityDelay();
@@ -435,6 +449,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
     public final ContentSearchCluster getSearch() { return search; }
 
     public Redundancy redundancy() { return redundancy; }
+
     public ContentCluster setRedundancy(Redundancy redundancy) {
         this.redundancy = redundancy;
         return this;