Change config for cluster controller to use min groups up

max-groups-allowed-down -> min-group-up-ratio

5 files changed, 105 insertions, 83 deletions

diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
index d870ec0e6a9..c9e71e7b58d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
@@ -16,7 +16,7 @@ import org.w3c.dom.Element;
 import java.util.Optional;
 
 /**
- * Config generation for common parameters for all fleet controllers.
+ * Config generation for parameters for fleet controllers.
  */
 public class ClusterControllerConfig extends AnyConfigProducer implements FleetcontrollerConfig.Producer {
 
@@ -49,25 +49,13 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                 clusterControllerTuning = tuning.child("cluster-controller");
             }
 
+            var numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups();
             var tuningConfig = new ClusterControllerTuningBuilder(clusterControllerTuning,
                                                                   minNodeRatioPerGroup,
-                                                                  bucketSplittingMinimumBits)
+                                                                  bucketSplittingMinimumBits,
+                                                                  allowMoreThanOneContentGroupDown,
+                                                                  numberOfLeafGroups)
                     .build();
-            if (ancestor instanceof ContentCluster) {
-                int numberOfLeafGroups = ((ContentCluster) ancestor).getRootGroup().getNumberOfLeafGroups();
-                if (tuningConfig.maxGroupsAllowedDown().isPresent()) {
-                    Integer maxGroupsAllowedDown = tuningConfig.maxGroupsAllowedDown().get();
-                    if (deployState.zone().environment().isProduction() && (maxGroupsAllowedDown > numberOfLeafGroups))
-                        throw new IllegalArgumentException("Cannot set max-groups-allowed-down (" + maxGroupsAllowedDown +
-                                                                   ") larger than number of groups (" + numberOfLeafGroups + ")");
-                } else {
-                    // Reduce to numberOfLeafGroups for tests or in environments where number of groups are reduced by policy (dev, test, staging, perf)
-                    tuningConfig = tuningConfig.withMaxGroupsAllowedDown(numberOfLeafGroups);
-                }
-            } else {
-                // Reduce to 1 for tests (ancestor is a mock class)
-                tuningConfig = tuningConfig.withMaxGroupsAllowedDown(1);
-            }
 
             return new ClusterControllerConfig(ancestor,
                                                clusterName,
@@ -134,11 +122,13 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
         private final Optional<Double> minDistributorUpRatio;
         private final Optional<Double> minStorageUpRatio;
         private final Optional<Integer> minSplitBits;
-        final Optional<Integer> maxGroupsAllowedDown;
+        private final Optional<Integer> maxGroupsAllowedDown;
 
         ClusterControllerTuningBuilder(ModelElement tuning,
                                        Optional<Double> minNodeRatioPerGroup,
-                                       Optional<Integer> bucketSplittingMinimumBits) {
+                                       Optional<Integer> bucketSplittingMinimumBits,
+                                       boolean maxGroupsAllowedDown,
+                                       int numberOfLeafGroups) {
             this.minSplitBits = bucketSplittingMinimumBits;
             this.minNodeRatioPerGroup = minNodeRatioPerGroup;
             if (tuning == null) {
@@ -157,8 +147,23 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                 this.stableStateTimePeriod = Optional.ofNullable(tuning.childAsDuration("stable-state-period"));
                 this.minDistributorUpRatio = Optional.ofNullable(tuning.childAsDouble("min-distributor-up-ratio"));
                 this.minStorageUpRatio = Optional.ofNullable(tuning.childAsDouble("min-storage-up-ratio"));
-                this.maxGroupsAllowedDown = Optional.ofNullable(tuning.childAsInteger("max-groups-allowed-down"));
+                this.maxGroupsAllowedDown = maxGroupsAllowedDown(tuning, maxGroupsAllowedDown, numberOfLeafGroups);
+            }
+        }
+
+
+        private static Optional<Integer> maxGroupsAllowedDown(ModelElement tuning, boolean allowMoreThanOneContentGroupDown, int numberOfLeafGroups) {
+            var minGroupsUpRatio = tuning.childAsDouble("min-group-up-ratio");
+
+            if (minGroupsUpRatio != null) {
+                if (minGroupsUpRatio < 0.01 || minGroupsUpRatio > 1)
+                    throw new IllegalArgumentException("min-groups-up-ratio must be between 0.01 and 1, got " + minGroupsUpRatio);
+                double minGroupsUp = minGroupsUpRatio * numberOfLeafGroups;
+                var maxGroupsAllowedDown = Math.max(1, numberOfLeafGroups - (int) Math.ceil(minGroupsUp));
+                return allowMoreThanOneContentGroupDown ? Optional.of(maxGroupsAllowedDown) : Optional.empty();
             }
+
+            return Optional.empty();
         }
 
         private ClusterControllerTuning build() {
@@ -184,20 +189,6 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                                            Optional<Integer> maxGroupsAllowedDown,
                                            Optional<Double> minNodeRatioPerGroup,
                                            Optional<Integer> minSplitBits) {
-
-        public ClusterControllerTuning withMaxGroupsAllowedDown(int maxGroupsAllowedDown) {
-            return new ClusterControllerConfig.ClusterControllerTuning(
-                    initProgressTime,
-                    transitionTime,
-                    maxPrematureCrashes,
-                    stableStateTimePeriod,
-                    minDistributorUpRatio,
-                    minStorageUpRatio,
-                    Optional.of(maxGroupsAllowedDown),
-                    minNodeRatioPerGroup,
-                    minSplitBits);
-        }
-
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index f792ac3a591..728bd59a4a0 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -207,7 +207,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
                 docprocChain = docprocChain.trim();
             }
             if (docprocCluster != null && !docprocCluster.isEmpty()) {
-                if (!c.getSearch().hasIndexedCluster() && !c.getSearch().getIndexingDocproc().isPresent() &&
+                if (!c.getSearch().hasIndexedCluster() && c.getSearch().getIndexingDocproc().isEmpty() &&
                         docprocChain != null && !docprocChain.isEmpty()) {
                     c.getSearch().setupStreamingSearchIndexingDocProc();
                 }
@@ -455,7 +455,7 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
 
     @Override
     public void getConfig(MessagetyperouteselectorpolicyConfig.Builder builder) {
-        if ( ! getSearch().getIndexingDocproc().isPresent()) return;
+        if (getSearch().getIndexingDocproc().isEmpty()) return;
         DocumentProtocol.getConfig(builder, getConfigId());
     }
 
diff --git a/config-model/src/main/resources/schema/content.rnc b/config-model/src/main/resources/schema/content.rnc
index a73236454c6..9705e2d8f5f 100644
--- a/config-model/src/main/resources/schema/content.rnc
+++ b/config-model/src/main/resources/schema/content.rnc
@@ -82,7 +82,7 @@ ClusterControllerTuning = element cluster-controller {
     element stable-state-period {  xsd:string { pattern = "([0-9\.]+)\s*([a-z]+)?" }  }? &
     element min-distributor-up-ratio { xsd:double }? &
     element min-storage-up-ratio { xsd:double }? &
-    element max-groups-allowed-down { xsd:nonNegativeInteger }?
+    element min-group-up-ratio { xsd:double }?
 }
 
 DispatchTuning = element dispatch {
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 19fe9e0038d..730736a128d 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -2375,7 +2375,7 @@ public class ModelProvisioningTest {
                         "    <nodes count='4' groups='4'/>" +
                         "    <tuning>" +
                         "      <cluster-controller>" +
-                        "        <max-groups-allowed-down>2</max-groups-allowed-down>" +
+                        "        <min-group-up-ratio>0.5</min-group-up-ratio>" +
                         "      </cluster-controller>" +
                         "    </tuning>" +
                         "  </content>" +
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 73bbd6ee464..7034a6878e1 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1422,51 +1422,30 @@ public class ContentClusterTest extends ContentBaseTest {
     }
 
     @Test
-    void testAllow2GroupsDown() {
-        String services = "<?xml version='1.0' encoding='UTF-8' ?>" +
-                "<services version='1.0'>" +
-                "  <container id='default' version='1.0' />" +
-                "  <content id='storage' version='1.0'>" +
-                "    <redundancy>4</redundancy>" +
-                "    <documents>" +
-                "      <document mode='index' type='type1' />" +
-                "    </documents>" +
-                "  <group name='root'>" +
-                "    <distribution partitions='1|1|1|*'/>" +
-                "    <group name='g-1' distribution-key='0'>" +
-                "      <node hostalias='mockhost' distribution-key='0'/>" +
-                "    </group>" +
-                "    <group name='g-2' distribution-key='1'>" +
-                "      <node hostalias='mockhost' distribution-key='1'/>" +
-                "    </group>" +
-                "    <group name='g-3' distribution-key='2'>" +
-                "      <node hostalias='mockhost' distribution-key='2'/>" +
-                "    </group>" +
-                "    <group name='g-4' distribution-key='3'>" +
-                "      <node hostalias='mockhost' distribution-key='3'/>" +
-                "    </group>" +
-                "  </group>" +
-                "    <tuning>" +
-                "      <cluster-controller>" +
-                "        <max-groups-allowed-down>2</max-groups-allowed-down>" +
-                "      </cluster-controller>" +
-                "    </tuning>" +
-                "    <engine>" +
-                "      <proton>" +
-                "        <searchable-copies>4</searchable-copies>" +
-                "      </proton>" +
-                "    </engine>" +
-                "  </content>" +
-                " </services>";
-        VespaModel model = createEnd2EndOneNode(new TestProperties().setAllowMoreThanOneContentGroupDown(true), services);
-
-        var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder();
-        model.getConfig(fleetControllerConfigBuilder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer");
-        assertEquals(2, fleetControllerConfigBuilder.build().max_number_of_groups_allowed_to_be_down());
-    }
-
-    private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled)
-    {
+    void testGroupsAllowedToBeDown() {
+        assertMaxAllowedGroupsDown(1, 0.5, 1);
+        assertMaxAllowedGroupsDown(2, 0.5, 1);
+        assertMaxAllowedGroupsDown(3, 0.5, 1);
+        assertMaxAllowedGroupsDown(4, 0.5, 2);
+        assertMaxAllowedGroupsDown(5, 0.5, 2);
+        assertMaxAllowedGroupsDown(6, 0.5, 3);
+
+        assertMaxAllowedGroupsDown(1, 0.33, 1);
+        assertMaxAllowedGroupsDown(2, 0.33, 1);
+        assertMaxAllowedGroupsDown(3, 0.33, 2);
+        assertMaxAllowedGroupsDown(4, 0.33, 2);
+        assertMaxAllowedGroupsDown(5, 0.33, 3);
+        assertMaxAllowedGroupsDown(6, 0.33, 4);
+
+        assertMaxAllowedGroupsDown(1, 0.67, 1);
+        assertMaxAllowedGroupsDown(2, 0.67, 1);
+        assertMaxAllowedGroupsDown(3, 0.67, 1);
+        assertMaxAllowedGroupsDown(4, 0.67, 1);
+        assertMaxAllowedGroupsDown(5, 0.67, 1);
+        assertMaxAllowedGroupsDown(6, 0.67, 1);
+    }
+
+    private void assertIndexingDocprocEnabled(boolean indexed, boolean force, boolean expEnabled) {
         String services = "<?xml version='1.0' encoding='UTF-8' ?>" +
                 "<services version='1.0'>" +
                 "  <container id='default' version='1.0'>" +
@@ -1503,4 +1482,56 @@ public class ContentClusterTest extends ContentBaseTest {
         assertIndexingDocprocEnabled(false, true, true);
     }
 
+    private void assertMaxAllowedGroupsDown(int groupCount, double minGroupUpRatio, int expectedMaxAllowedGroupsDown) {
+        var services = servicesWithGroups(groupCount, minGroupUpRatio);
+        var model = createEnd2EndOneNode(new TestProperties().setAllowMoreThanOneContentGroupDown(true), services);
+
+        var fleetControllerConfigBuilder = new FleetcontrollerConfig.Builder();
+        model.getConfig(fleetControllerConfigBuilder, "admin/cluster-controllers/0/components/clustercontroller-storage-configurer");
+        var config = fleetControllerConfigBuilder.build();
+
+        assertEquals(expectedMaxAllowedGroupsDown, config.max_number_of_groups_allowed_to_be_down());
+    }
+
+    private String servicesWithGroups(int groupCount, double minGroupUpRatio) {
+        String services = String.format("<?xml version='1.0' encoding='UTF-8' ?>" +
+                "<services version='1.0'>" +
+                "  <container id='default' version='1.0' />" +
+                "  <content id='storage' version='1.0'>" +
+                "    <redundancy>%d</redundancy>" +
+                "    <documents>" +
+                "      <document mode='index' type='type1' />" +
+                "    </documents>" +
+                "  <group name='root'>", groupCount);
+        String distribution = switch (groupCount) {
+            case 1, 2 -> "    <distribution partitions='1|*'/>";
+            case 3 -> "    <distribution partitions='1|1|*'/>";
+            case 4 -> "    <distribution partitions='1|1|1|*'/>";
+            case 5 -> "    <distribution partitions='1|1|1|1|*'/>";
+            case 6 -> "    <distribution partitions='1|1|1|1|1|*'/>";
+            default -> throw new IllegalArgumentException("Does not support groupCount > 6");
+        };
+        services += distribution;
+        for (int i = 0; i < groupCount; i++) {
+            services += String.format("    <group name='g-%d' distribution-key='%d'>" +
+                                              "      <node hostalias='mockhost' distribution-key='%d'/>" +
+                                              "    </group>",
+                                      i, i, i);
+        }
+        return services +
+                String.format("  </group>" +
+                "    <tuning>" +
+                "      <cluster-controller>" +
+                "        <min-group-up-ratio>%f</min-group-up-ratio>" +
+                "      </cluster-controller>" +
+                "    </tuning>" +
+                "    <engine>" +
+                "      <proton>" +
+                "        <searchable-copies>%d</searchable-copies>" +
+                "      </proton>" +
+                "    </engine>" +
+                "  </content>" +
+                " </services>", minGroupUpRatio, groupCount);
+    }
+
 }