Merge pull request #19434 from vespa-engine/vekterli/add-feature-flag-for-ignoring-merge-queue-limit

Add feature flag for ignoring merge queue limit for forwarded (chained) merges [run-systemtest]

6 files changed, 42 insertions, 4 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 694193923c5..528b41efb6a 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -83,6 +83,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"baldersheim"}) default int largeRankExpressionLimit() { return 8192; }
         @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); }
+        @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 570638e7e93..7c3d8e4763f 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -57,6 +57,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private int numDistributorStripes = 0;
     private int maxConcurrentMergesPerNode = 16;
     private int maxMergeQueueSize = 1024;
+    private boolean ignoreMergeQueueLimit = false;
     private int largeRankExpressionLimit = 8192;
     private boolean allowDisableMtls = true;
     private List<X509Certificate> operatorCertificates = Collections.emptyList();
@@ -104,6 +105,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; }
     @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; }
     @Override public int maxMergeQueueSize() { return maxMergeQueueSize; }
+    @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; }
     @Override public double resourceLimitDisk() { return resourceLimitDisk; }
     @Override public double resourceLimitMemory() { return resourceLimitMemory; }
     @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; }
@@ -165,6 +167,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setIgnoreMergeQueueLimit(boolean ignoreMergeQueueLimit) {
+        this.ignoreMergeQueueLimit = ignoreMergeQueueLimit;
+        return this;
+    }
+
     public TestProperties setDefaultTermwiseLimit(double limit) {
         defaultTermwiseLimit = limit;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
index e89d45e8b83..7acae9b32f9 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java
@@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer {
     private Integer maxMergesPerNode;
     private Integer queueSize;
     private Integer bucketDBStripeBits;
+    private Boolean ignoreMergeQueueLimit;
 
     private StorServerProducer setMaxMergesPerNode(Integer value) {
         if (value != null) {
@@ -54,6 +55,7 @@ public class StorServerProducer implements StorServerConfig.Producer {
         this.clusterName = clusterName;
         maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode();
         queueSize = featureFlags.maxMergeQueueSize();
+        ignoreMergeQueueLimit = featureFlags.ignoreMergeQueueLimit();
     }
 
     @Override
@@ -73,5 +75,8 @@ public class StorServerProducer implements StorServerConfig.Producer {
         if (bucketDBStripeBits != null) {
             builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits);
         }
+        if (ignoreMergeQueueLimit != null) {
+            builder.disable_queue_limits_for_chained_merges(ignoreMergeQueueLimit);
+        }
     }
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 9a681003293..fd85ea3d024 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -118,6 +118,7 @@ public class StorageClusterTest {
         StorServerConfig config = new StorServerConfig(builder);
         assertEquals(16, config.max_merges_per_node());
         assertEquals(1024, config.max_merge_queue_size());
+        assertFalse(config.disable_queue_limits_for_chained_merges());
     }
 
     @Test
@@ -134,17 +135,30 @@ public class StorageClusterTest {
         assertEquals(1024, config.max_merges_per_node());
         assertEquals(1024*10, config.max_merge_queue_size());
     }
-    @Test
-    public void testMergeFeatureFlags() {
+
+    private StorServerConfig configFromProperties(TestProperties properties) {
         StorServerConfig.Builder builder = new StorServerConfig.Builder();
-        parse(cluster("foofighters", ""), new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)).getConfig(builder);
+        parse(cluster("foofighters", ""), properties).getConfig(builder);
+        return new StorServerConfig(builder);
+    }
 
-        StorServerConfig config = new StorServerConfig(builder);
+    @Test
+    public void testMergeFeatureFlags() {
+        var config = configFromProperties(new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37));
         assertEquals(37, config.max_merges_per_node());
         assertEquals(1919, config.max_merge_queue_size());
     }
 
     @Test
+    public void ignore_merge_queue_limit_can_be_controlled_by_feature_flag() {
+        var config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(true));
+        assertTrue(config.disable_queue_limits_for_chained_merges());
+
+        config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(false));
+        assertFalse(config.disable_queue_limits_for_chained_merges());
+    }
+
+    @Test
     public void testVisitors() {
         StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder();
         parse(cluster("bees",
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index f4c805bd976..badfb3ad931 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -182,6 +182,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean requireConnectivityCheck;
         private final int maxConcurrentMergesPerContentNode;
         private final int maxMergeQueueSize;
+        private final boolean ignoreMergeQueueLimit;
         private final int largeRankExpressionLimit;
         private final double resourceLimitDisk;
         private final double resourceLimitMemory;
@@ -213,6 +214,7 @@ public class ModelContextImpl implements ModelContext {
             this.requireConnectivityCheck = flagValue(source, appId, Flags.REQUIRE_CONNECTIVITY_CHECK);
             this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE);
             this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE);
+            this.ignoreMergeQueueLimit = flagValue(source, appId, Flags.IGNORE_MERGE_QUEUE_LIMIT);
             this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK);
             this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY);
             this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP);
@@ -245,6 +247,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean requireConnectivityCheck() { return requireConnectivityCheck; }
         @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; }
         @Override public int maxMergeQueueSize() { return maxMergeQueueSize; }
+        @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; }
         @Override public double resourceLimitDisk() { return resourceLimitDisk; }
         @Override public double resourceLimitMemory() { return resourceLimitMemory; }
         @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index fa7f6eedace..74d187368bc 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -197,6 +197,14 @@ public class Flags {
             "Takes effect at redeploy",
             ZONE_ID, APPLICATION_ID);
 
+    public static final UnboundBooleanFlag IGNORE_MERGE_QUEUE_LIMIT = defineFeatureFlag(
+            "ignore-merge-queue-limit", false,
+            List.of("vekterli", "geirst"), "2021-10-06", "2021-12-01",
+            "Specifies if merges that are forwarded (chained) from another content node are always " +
+                    "allowed to be enqueued even if the queue is otherwise full.",
+            "Takes effect at redeploy",
+            ZONE_ID, APPLICATION_ID);
+
     public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag(
             "large-rank-expression-limit", 8192,
             List.of("baldersheim"), "2021-06-09", "2021-11-01",