From e68f0aa6f7cb59440f638dcb87e0d27bf53ce9a8 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Tue, 25 Jan 2022 14:28:47 +0100 Subject: Add feature flag for merge throttling policy Two options; `STATIC` (default) and `DYNAMIC`. Does not currently set any window size configs based on the existing feature flag for max pending, so the config defaults will be used for these when dynamic throttling is enabled. --- .../com/yahoo/config/model/deploy/TestProperties.java | 7 +++++++ .../content/storagecluster/StorServerProducer.java | 12 ++++++++++++ .../yahoo/vespa/model/content/StorageClusterTest.java | 19 +++++++++++++++++++ 3 files changed, 38 insertions(+) (limited to 'config-model') diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 1478854aecf..3aadf30e2f9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -71,6 +71,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; + private String mergeThrottlingPolicy = "STATIC"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -121,6 +122,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -313,6 +315,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMergeThrottlingPolicy(String policy) { + this.mergeThrottlingPolicy = policy; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 6f9166f4493..2fca964a995 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; + private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -50,10 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } + private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { + try { + return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); + } catch (Throwable t) { + return StorServerConfig.Merge_throttling_policy.Type.STATIC; + } + } + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); + mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -73,5 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } + // TODO set throttle policy params based on existing or separate flags + builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 06b907ca2c2..2953b0cb32d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -159,6 +159,25 @@ public class StorageClusterTest { assertEquals(1919, config.max_merge_queue_size()); } + @Test + public void merge_throttling_policy_config_defaults_to_static() { + var config = configFromProperties(new TestProperties()); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + + @Test + public void merge_throttling_policy_config_is_derived_from_flag() { + var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); + + // Invalid enum values fall back to the default + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); -- cgit v1.2.3