diff options
author | Tor Brede Vekterli <vekterli@verizonmedia.com> | 2021-10-06 14:26:31 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-10-06 14:26:31 +0200 |
commit | 87c26111db1a8071ad58a0d78fc455a3e765a3fa (patch) | |
tree | 8aed27f33ef883af99d22df5d795adc6b408c606 | |
parent | 03e71008c79557781537baf84c91bd00e08aa203 (diff) | |
parent | 3ce2a1138cd90d83d1eeb35c1b5f2503872327fe (diff) |
Merge pull request #19434 from vespa-engine/vekterli/add-feature-flag-for-ignoring-merge-queue-limit
Add feature flag for ignoring merge queue limit for forwarded (chained) merges [run-systemtest]
6 files changed, 42 insertions, 4 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 694193923c5..528b41efb6a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -83,6 +83,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int largeRankExpressionLimit() { return 8192; } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxConcurrentMergesPerNode() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int maxMergeQueueSize() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean ignoreMergeQueueLimit() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean containerDumpHeapOnShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double containerShutdownTimeout() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 570638e7e93..7c3d8e4763f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -57,6 +57,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int numDistributorStripes = 0; private int maxConcurrentMergesPerNode = 16; private int maxMergeQueueSize = 1024; + private boolean ignoreMergeQueueLimit = false; private int largeRankExpressionLimit = 8192; private boolean allowDisableMtls = true; private List<X509Certificate> operatorCertificates = Collections.emptyList(); @@ -104,6 +105,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int largeRankExpressionLimit() { return largeRankExpressionLimit; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } + @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } @@ -165,6 +167,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setIgnoreMergeQueueLimit(boolean ignoreMergeQueueLimit) { + this.ignoreMergeQueueLimit = ignoreMergeQueueLimit; + return this; + } + public TestProperties setDefaultTermwiseLimit(double limit) { defaultTermwiseLimit = limit; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index e89d45e8b83..7acae9b32f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; + private Boolean ignoreMergeQueueLimit; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -54,6 +55,7 @@ public class StorServerProducer implements StorServerConfig.Producer { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); + ignoreMergeQueueLimit = featureFlags.ignoreMergeQueueLimit(); } @Override @@ -73,5 +75,8 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } + if (ignoreMergeQueueLimit != null) { + builder.disable_queue_limits_for_chained_merges(ignoreMergeQueueLimit); + } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 9a681003293..fd85ea3d024 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -118,6 +118,7 @@ public class StorageClusterTest { StorServerConfig config = new StorServerConfig(builder); assertEquals(16, config.max_merges_per_node()); assertEquals(1024, config.max_merge_queue_size()); + assertFalse(config.disable_queue_limits_for_chained_merges()); } @Test @@ -134,17 +135,30 @@ public class StorageClusterTest { assertEquals(1024, config.max_merges_per_node()); assertEquals(1024*10, config.max_merge_queue_size()); } - @Test - public void testMergeFeatureFlags() { + + private StorServerConfig configFromProperties(TestProperties properties) { StorServerConfig.Builder builder = new StorServerConfig.Builder(); - parse(cluster("foofighters", ""), new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)).getConfig(builder); + parse(cluster("foofighters", ""), properties).getConfig(builder); + return new StorServerConfig(builder); + } - StorServerConfig config = new StorServerConfig(builder); + @Test + public void testMergeFeatureFlags() { + var config = configFromProperties(new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37)); assertEquals(37, config.max_merges_per_node()); assertEquals(1919, config.max_merge_queue_size()); } @Test + public void ignore_merge_queue_limit_can_be_controlled_by_feature_flag() { + var config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(true)); + assertTrue(config.disable_queue_limits_for_chained_merges()); + + config = configFromProperties(new TestProperties().setIgnoreMergeQueueLimit(false)); + assertFalse(config.disable_queue_limits_for_chained_merges()); + } + + @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); parse(cluster("bees", diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index f4c805bd976..badfb3ad931 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -182,6 +182,7 @@ public class ModelContextImpl implements ModelContext { private final boolean requireConnectivityCheck; private final int maxConcurrentMergesPerContentNode; private final int maxMergeQueueSize; + private final boolean ignoreMergeQueueLimit; private final int largeRankExpressionLimit; private final double resourceLimitDisk; private final double resourceLimitMemory; @@ -213,6 +214,7 @@ public class ModelContextImpl implements ModelContext { this.requireConnectivityCheck = flagValue(source, appId, Flags.REQUIRE_CONNECTIVITY_CHECK); this.maxConcurrentMergesPerContentNode = flagValue(source, appId, Flags.MAX_CONCURRENT_MERGES_PER_NODE); this.maxMergeQueueSize = flagValue(source, appId, Flags.MAX_MERGE_QUEUE_SIZE); + this.ignoreMergeQueueLimit = flagValue(source, appId, Flags.IGNORE_MERGE_QUEUE_LIMIT); this.resourceLimitDisk = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_DISK); this.resourceLimitMemory = flagValue(source, appId, PermanentFlags.RESOURCE_LIMIT_MEMORY); this.minNodeRatioPerGroup = flagValue(source, appId, Flags.MIN_NODE_RATIO_PER_GROUP); @@ -245,6 +247,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean requireConnectivityCheck() { return requireConnectivityCheck; } @Override public int maxConcurrentMergesPerNode() { return maxConcurrentMergesPerContentNode; } @Override public int maxMergeQueueSize() { return maxMergeQueueSize; } + @Override public boolean ignoreMergeQueueLimit() { return ignoreMergeQueueLimit; } @Override public double resourceLimitDisk() { return resourceLimitDisk; } @Override public double resourceLimitMemory() { return resourceLimitMemory; } @Override public double minNodeRatioPerGroup() { return minNodeRatioPerGroup; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index fa7f6eedace..74d187368bc 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -197,6 +197,14 @@ public class Flags { "Takes effect at redeploy", ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag IGNORE_MERGE_QUEUE_LIMIT = defineFeatureFlag( + "ignore-merge-queue-limit", false, + List.of("vekterli", "geirst"), "2021-10-06", "2021-12-01", + "Specifies if merges that are forwarded (chained) from another content node are always " + + "allowed to be enqueued even if the queue is otherwise full.", + "Takes effect at redeploy", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag LARGE_RANK_EXPRESSION_LIMIT = defineIntFlag( "large-rank-expression-limit", 8192, List.of("baldersheim"), "2021-06-09", "2021-11-01", |