Merge pull request #26749 from vespa-engine/hmusum/add-flag-for-allowing-more-than-one-content-group-down

Add flag for allowing more than one content group down

6 files changed, 47 insertions, 23 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index fef2354c452..7f2dd4b6acd 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -113,6 +113,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"tokle, bjorncs"}, removeAfter = "8.108") default boolean enableDataPlaneFilter() { return true; }
         @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; }
         @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; }
+        @ModelFeatureFlag(owners = {"hmusum"}) default boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return false; }
 
         //Below are all flags that must be kept until 7 is out of the door
         @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
index 8ec4ae35658..201e0b5693a 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ClusterControllerConfig.java
@@ -14,8 +14,6 @@ import org.w3c.dom.Element;
 
 /**
  * Config generation for common parameters for all fleet controllers.
- *
- * TODO: Author
  */
 public class ClusterControllerConfig extends AnyConfigProducer implements FleetcontrollerConfig.Producer {
 
@@ -23,11 +21,16 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
         private final String clusterName;
         private final ModelElement clusterElement;
         private final ResourceLimits resourceLimits;
+        private final boolean allowMoreThanOneContentGroupDown;
 
-        public Builder(String clusterName, ModelElement clusterElement, ResourceLimits resourceLimits) {
+        public Builder(String clusterName,
+                       ModelElement clusterElement,
+                       ResourceLimits resourceLimits,
+                       boolean allowMoreThanOneContentGroupDown) {
             this.clusterName = clusterName;
             this.clusterElement = clusterElement;
             this.resourceLimits = resourceLimits;
+            this.allowMoreThanOneContentGroupDown = allowMoreThanOneContentGroupDown;
         }
 
         @Override
@@ -53,13 +56,15 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                         tuning.childAsDouble("min-storage-up-ratio"),
                         bucketSplittingMinimumBits,
                         minNodeRatioPerGroup,
-                        resourceLimits);
+                        resourceLimits,
+                        allowMoreThanOneContentGroupDown);
             } else {
                 return new ClusterControllerConfig(ancestor, clusterName,
                         null, null, null, null, null, null,
                         bucketSplittingMinimumBits,
                         minNodeRatioPerGroup,
-                        resourceLimits);
+                        resourceLimits,
+                        allowMoreThanOneContentGroupDown);
             }
         }
     }
@@ -74,6 +79,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
     private final Integer minSplitBits;
     private final Double minNodeRatioPerGroup;
     private final ResourceLimits resourceLimits;
+    private final boolean allowMoreThanOneContentGroupDown;
 
     // TODO refactor; too many args
     private ClusterControllerConfig(TreeConfigProducer<?> parent,
@@ -86,7 +92,8 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
                                     Double minStorageUpRatio,
                                     Integer minSplitBits,
                                     Double minNodeRatioPerGroup,
-                                    ResourceLimits resourceLimits) {
+                                    ResourceLimits resourceLimits,
+                                    boolean allowMoreThanOneContentGroupDown) {
         super(parent, "fleetcontroller");
 
         this.clusterName = clusterName;
@@ -99,6 +106,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
         this.minSplitBits = minSplitBits;
         this.minNodeRatioPerGroup = minNodeRatioPerGroup;
         this.resourceLimits = resourceLimits;
+        this.allowMoreThanOneContentGroupDown = allowMoreThanOneContentGroupDown;
     }
 
     @Override
@@ -141,6 +149,7 @@ public class ClusterControllerConfig extends AnyConfigProducer implements Fleetc
             builder.min_node_ratio_per_group(minNodeRatioPerGroup);
         }
         resourceLimits.getConfig(builder);
+        builder.max_number_of_groups_allowed_to_be_down(allowMoreThanOneContentGroupDown ? 1 : -1);
     }
 
 }
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 7f4fc4cd89d..217c26516a9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -127,7 +127,8 @@ public class ContentCluster extends TreeConfigProducer<AnyConfigProducer> implem
                     .build(contentElement);
             c.clusterControllerConfig = new ClusterControllerConfig.Builder(clusterId,
                                                                             contentElement,
-                                                                            resourceLimits.getClusterControllerLimits())
+                                                                            resourceLimits.getClusterControllerLimits(),
+                                                                            deployState.featureFlags().allowMoreThanOneContentGroupDown(new ClusterSpec.Id(clusterId)))
                     .build(deployState, c, contentElement.getXml());
             c.search = new ContentSearchCluster.Builder(documentDefinitions,
                                                         globallyDistributedDocuments,
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
index 1e6847a47be..1f8dea41a3e 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/FleetControllerClusterTest.java
@@ -27,7 +27,8 @@ public class FleetControllerClusterTest {
                                                    new ClusterResourceLimits.Builder(false,
                                                                                      featureFlags.resourceLimitDisk(),
                                                                                      featureFlags.resourceLimitMemory())
-                                                           .build(clusterElement).getClusterControllerLimits())
+                                                           .build(clusterElement).getClusterControllerLimits(),
+                                                   false)
                 .build(root.getDeployState(), root, clusterElement.getXml());
     }
 
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 7a2377594a1..62431ce4c06 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -22,10 +22,10 @@ import com.yahoo.config.provision.AthenzDomain;
 import com.yahoo.config.provision.CloudAccount;
 import com.yahoo.config.provision.ClusterSpec;
 import com.yahoo.config.provision.DockerImage;
+import com.yahoo.config.provision.HostName;
 import com.yahoo.config.provision.TenantName;
 import com.yahoo.config.provision.Zone;
 import com.yahoo.container.jdisc.secretstore.SecretStore;
-import com.yahoo.config.provision.HostName;
 import com.yahoo.vespa.config.server.tenant.SecretStoreExternalIdRetriever;
 import com.yahoo.vespa.flags.FetchVector;
 import com.yahoo.vespa.flags.FlagSource;
@@ -33,7 +33,6 @@ import com.yahoo.vespa.flags.Flags;
 import com.yahoo.vespa.flags.PermanentFlags;
 import com.yahoo.vespa.flags.StringFlag;
 import com.yahoo.vespa.flags.UnboundFlag;
-
 import java.io.File;
 import java.net.URI;
 import java.security.cert.X509Certificate;
@@ -41,7 +40,7 @@ import java.util.List;
 import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
-import java.util.function.ToIntFunction;
+import java.util.function.Predicate;
 
 import static com.yahoo.config.provision.NodeResources.Architecture;
 import static com.yahoo.vespa.config.server.ConfigServerSpec.fromConfig;
@@ -174,7 +173,7 @@ public class ModelContextImpl implements ModelContext {
         private final double feedNiceness;
         private final List<String> allowedAthenzProxyIdentities;
         private final int maxActivationInhibitedOutOfSyncGroups;
-        private final ToIntFunction<ClusterSpec.Type> jvmOmitStackTraceInFastThrow;
+        private final Predicate<ClusterSpec.Type> jvmOmitStackTraceInFastThrow;
         private final double resourceLimitDisk;
         private final double resourceLimitMemory;
         private final double minNodeRatioPerGroup;
@@ -204,6 +203,7 @@ public class ModelContextImpl implements ModelContext {
         private final int heapPercentage;
         private final boolean enableGlobalPhase;
         private final String summaryDecodePolicy;
+        private final Predicate<ClusterSpec.Id> allowMoreThanOneContentGroupDown;
 
         public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
             this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -219,7 +219,7 @@ public class ModelContextImpl implements ModelContext {
             this.mbus_network_threads = flagValue(source, appId, version, Flags.MBUS_NUM_NETWORK_THREADS);
             this.allowedAthenzProxyIdentities = flagValue(source, appId, version, Flags.ALLOWED_ATHENZ_PROXY_IDENTITIES);
             this.maxActivationInhibitedOutOfSyncGroups = flagValue(source, appId, version, Flags.MAX_ACTIVATION_INHIBITED_OUT_OF_SYNC_GROUPS);
-            this.jvmOmitStackTraceInFastThrow = type -> flagValueAsInt(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW);
+            this.jvmOmitStackTraceInFastThrow = type -> flagValue(source, appId, version, type, PermanentFlags.JVM_OMIT_STACK_TRACE_IN_FAST_THROW);
             this.resourceLimitDisk = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_DISK);
             this.resourceLimitMemory = flagValue(source, appId, version, PermanentFlags.RESOURCE_LIMIT_MEMORY);
             this.minNodeRatioPerGroup = flagValue(source, appId, version, Flags.MIN_NODE_RATIO_PER_GROUP);
@@ -250,6 +250,7 @@ public class ModelContextImpl implements ModelContext {
             this.heapPercentage = flagValue(source, appId, version, PermanentFlags.HEAP_SIZE_PERCENTAGE);
             this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE);
             this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY);
+            this.allowMoreThanOneContentGroupDown = clusterId -> flagValue(source, appId, version, clusterId, Flags.ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN);
         }
 
         @Override public int heapSizePercentage() { return heapPercentage; }
@@ -270,7 +271,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public List<String> allowedAthenzProxyIdentities() { return allowedAthenzProxyIdentities; }
         @Override public int maxActivationInhibitedOutOfSyncGroups() { return maxActivationInhibitedOutOfSyncGroups; }
         @Override public String jvmOmitStackTraceInFastThrowOption(ClusterSpec.Type type) {
-            return translateJvmOmitStackTraceInFastThrowIntToString(jvmOmitStackTraceInFastThrow, type);
+            return translateJvmOmitStackTraceInFastThrowToString(jvmOmitStackTraceInFastThrow, type);
         }
         @Override public double resourceLimitDisk() { return resourceLimitDisk; }
         @Override public double resourceLimitMemory() { return resourceLimitMemory; }
@@ -304,6 +305,7 @@ public class ModelContextImpl implements ModelContext {
         }
         @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; }
         @Override public boolean enableGlobalPhase() { return enableGlobalPhase; }
+        @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown.test(id); }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
@@ -331,17 +333,21 @@ public class ModelContextImpl implements ModelContext {
                        .boxedValue();
         }
 
-        static int flagValueAsInt(FlagSource source,
-                                  ApplicationId appId,
-                                  Version version,
-                                  ClusterSpec.Type clusterType,
-                                  UnboundFlag<? extends Boolean, ?, ?> flag) {
-            return flagValue(source, appId, version, clusterType, flag) ? 1 : 0;
+        private static <V> V flagValue(FlagSource source,
+                                       ApplicationId appId,
+                                       Version vespaVersion,
+                                       ClusterSpec.Id clusterId,
+                                       UnboundFlag<? extends V, ?, ?> flag) {
+            return flag.bindTo(source)
+                       .with(FetchVector.Dimension.APPLICATION_ID, appId.serializedForm())
+                       .with(FetchVector.Dimension.CLUSTER_ID, clusterId.value())
+                       .with(FetchVector.Dimension.VESPA_VERSION, vespaVersion.toFullString())
+                       .boxedValue();
         }
 
-        private String translateJvmOmitStackTraceInFastThrowIntToString(ToIntFunction<ClusterSpec.Type> function,
-                                                                        ClusterSpec.Type clusterType) {
-            return function.applyAsInt(clusterType) == 1 ? "" : "-XX:-OmitStackTraceInFastThrow";
+        private String translateJvmOmitStackTraceInFastThrowToString(Predicate<ClusterSpec.Type> function,
+                                                                     ClusterSpec.Type clusterType) {
+            return function.test(clusterType) ? "" : "-XX:-OmitStackTraceInFastThrow";
         }
 
     }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index d55b58a1728..ea17d9967ae 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -398,6 +398,12 @@ public class Flags {
             "Whether to enable CrowdStrike.", "Takes effect on next host admin tick",
             HOSTNAME);
 
+    public static final UnboundBooleanFlag ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN = defineFeatureFlag(
+            "allow-more-than-one-content-group-down", false, List.of("hmusum"), "2023-04-14", "2023-06-14",
+            "Whether to enable possible configuration of letting more than one content group down",
+            "Takes effect at redeployment",
+            HOSTNAME);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,