From 3ce2a1138cd90d83d1eeb35c1b5f2503872327fe Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 6 Oct 2021 13:52:44 +0200 Subject: Add feature flag for ignoring merge queue limit for forwarded (chained) merges --- .../yahoo/config/model/deploy/TestProperties.java | 7 +++++++ .../content/storagecluster/StorServerProducer.java | 5 +++++ .../vespa/model/content/StorageClusterTest.java | 22 ++++++++++++++++++---- 3 files changed, 30 insertions(+), 4 deletions(-) (limited to 'config-model/src') diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 570638e7e93..7c3d8e4763f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -57,6 +57,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int numDistributorStripes = 0; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 1024; + private boolean ignoreMergeQueueLimit = false; private int largeRankExpressionLimit = 8192; private boolean allowDisableMtls = true; private List operatorCertificates = Collections.emptyList(); @@ -104,6 +105,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } + @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @@ -165,6 +167,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setIgnoreMergeQueueLimit(boolean ignoreMergeQueueLimit) { + this.ignoreMergeQueueLimit = ignoreMergeQueueLimit; + return this; + } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index e89d45e8b83..7acae9b32f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; + private Boolean ignoreMergeQueueLimit; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -54,6 +55,7 @@ public class StorServerProducer implements StorServerConfig.Producer { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); + ignoreMergeQueueLimit = featureFlags.ignoreMergeQueueLimit(); } @Override @@ -73,5 +75,8 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } + if (ignoreMergeQueueLimit != null) { + builder.disable_queue_limits_for_chained_merges(ignoreMergeQueueLimit); + } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 9a681003293..fd85ea3d024 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -118,6 +118,7 @@ public class StorageClusterTest { StorServerConfig config = new StorServerConfig(builder); assertEquals(16, config.max_merges_per_node()); assertEquals(1024, config.max_merge_queue_size()); + assertFalse(config.disable_queue_limits_for_chained_merges()); } @Test @@ -134,16 +135,29 @@ public class StorageClusterTest { assertEquals(1024, config.max_merges_per_node()); assertEquals(1024*10, config.max_merge_queue_size()); } - @Test - public void testMergeFeatureFlags() { + + private StorServerConfig configFromProperties(TestProperties properties) { StorServerConfig.Builder builder = new StorServerConfig.Builder(); - parse(cluster("foofighters", ""), new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)).getConfig(builder); + parse(cluster("foofighters", ""), properties).getConfig(builder); + return new StorServerConfig(builder); + } - StorServerConfig config = new StorServerConfig(builder); + @Test + public void testMergeFeatureFlags() { + var config = configFromProperties(new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)); assertEquals(37, config.max_merges_per_node()); assertEquals(1919, config.max_merge_queue_size()); } + @Test + public void ignore_merge_queue_limit_can_be_controlled_by_feature_flag() { + var config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(true)); + assertTrue(config.disable_queue_limits_for_chained_merges()); + + config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(false)); + assertFalse(config.disable_queue_limits_for_chained_merges()); + } + @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); -- cgit v1.2.3