Merge pull request #20947 from vespa-engine/vekterli/merge-throttling-policy-feature-flag-2

Vekterli/merge throttling policy feature flag 2 [run-systemtest]

6 files changed, 50 insertions, 0 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 672414f2d47..3bb6c7c0511 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -106,6 +106,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); }
         @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; }
         @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() {  throw new UnsupportedOperationException("TODO specify default value"); }
+        @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() {  throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; }
     }
 
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 1478854aecf..3aadf30e2f9 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -71,6 +71,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private int maxCompactBuffers = 1;
     private boolean failDeploymentWithInvalidJvmOptions = false;
     private String persistenceAsyncThrottling = "UNLIMITED";
+    private String mergeThrottlingPolicy = "STATIC";
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -121,6 +122,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public int maxCompactBuffers() { return maxCompactBuffers; }
     @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; }
     @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; }
+    @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; }
 
     public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) {
         this.maxUnCommittedMemory = maxUnCommittedMemory;
@@ -313,6 +315,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setMergeThrottlingPolicy(String policy) {
+        this.mergeThrottlingPolicy = policy;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
index 6f9166f4493..2fca964a995 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
@@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer {
     private Integer maxMergesPerNode;
     private Integer queueSize;
     private Integer bucketDBStripeBits;
+    private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType;
 
     private StorServerProducer setMaxMergesPerNode(Integer value) {
         if (value != null) {
@@ -50,10 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer {
         return this;
     }
 
+    private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) {
+        try {
+            return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType);
+        } catch (Throwable t) {
+            return StorServerConfig.Merge_throttling_policy.Type.STATIC;
+        }
+    }
+
     StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) {
         this.clusterName = clusterName;
         maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode();
         queueSize = featureFlags.maxMergeQueueSize();
+        mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy());
     }
 
     @Override
@@ -73,5 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer {
         if (bucketDBStripeBits != null) {
             builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits);
         }
+        // TODO set throttle policy params based on existing or separate flags
+        builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType));
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 06b907ca2c2..2953b0cb32d 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -160,6 +160,25 @@ public class StorageClusterTest {
     }
 
     @Test
+    public void merge_throttling_policy_config_defaults_to_static() {
+        var config = configFromProperties(new TestProperties());
+        assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type());
+    }
+
+    @Test
+    public void merge_throttling_policy_config_is_derived_from_flag() {
+        var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC"));
+        assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type());
+
+        config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC"));
+        assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type());
+
+        // Invalid enum values fall back to the default
+        config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE"));
+        assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type());
+    }
+
+    @Test
     public void testVisitors() {
         StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder();
         parse(cluster("bees",
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index fddb80610cd..89d3318d710 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext {
         private final List<String> ignoredHttpUserAgents;
         private final boolean enableServerOcspStapling;
         private final String persistenceAsyncThrottling;
+        private final String mergeThrottlingPolicy;
         private final boolean useQrserverServiceName;
 
         public FeatureFlags(FlagSource source, ApplicationId appId) {
@@ -241,6 +242,7 @@ public class ModelContextImpl implements ModelContext {
             this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS);
             this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING);
             this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING);
+            this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY);
             this.useQrserverServiceName =  flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME);
         }
 
@@ -283,6 +285,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; }
         @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; }
         @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; }
+        @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; }
         @Override public boolean useQrserverServiceName() { return useQrserverServiceName; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 30a7c2cd600..470bf9a0c1c 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -357,6 +357,14 @@ public class Flags {
             "Triggers restart, takes effect immediately",
             ZONE_ID, APPLICATION_ID);
 
+    public static final UnboundStringFlag MERGE_THROTTLING_POLICY = defineStringFlag(
+            "merge-throttling-policy", "STATIC",
+            List.of("vekterli"), "2022-01-25", "2022-05-01",
+            "Sets the policy used for merge throttling on the content nodes. " +
+            "Valid values: STATIC, DYNAMIC",
+            "Takes effect at redeployment",
+            ZONE_ID, APPLICATION_ID);
+
     public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag(
             "check-config-convergence-before-restart", false,
             List.of("hmusum"), "2022-01-16", "2022-02-16",