diff options
author | Tor Brede Vekterli <vekterli@yahooinc.com> | 2022-01-27 10:27:32 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-27 10:27:32 +0100 |
commit | c790fe3631168f68efcc8c9e8fe5a30e414517d2 (patch) | |
tree | 2bf9bcf56a1a3b7fcfc2f058ff343ec3d0df4725 | |
parent | 81aeddf945aadb85a916a8f31b07d65685d9a5bb (diff) | |
parent | 5bbde32b210752f601da9475edaeb3d76956abea (diff) |
Merge pull request #20947 from vespa-engine/vekterli/merge-throttling-policy-feature-flag-2
Vekterli/merge throttling policy feature flag 2 [run-systemtest]
6 files changed, 50 insertions, 0 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 672414f2d47..3bb6c7c0511 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -106,6 +106,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List<String> ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 1478854aecf..3aadf30e2f9 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -71,6 +71,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private String persistenceAsyncThrottling = "UNLIMITED"; + private String mergeThrottlingPolicy = "STATIC"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -121,6 +122,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -313,6 +315,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setMergeThrottlingPolicy(String policy) { + this.mergeThrottlingPolicy = policy; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 6f9166f4493..2fca964a995 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -32,6 +32,7 @@ public class StorServerProducer implements StorServerConfig.Producer { private Integer maxMergesPerNode; private Integer queueSize; private Integer bucketDBStripeBits; + private StorServerConfig.Merge_throttling_policy.Type.Enum mergeThrottlingPolicyType; private StorServerProducer setMaxMergesPerNode(Integer value) { if (value != null) { @@ -50,10 +51,19 @@ public class StorServerProducer implements StorServerConfig.Producer { return this; } + private static StorServerConfig.Merge_throttling_policy.Type.Enum toThrottlePolicyType(String policyType) { + try { + return StorServerConfig.Merge_throttling_policy.Type.Enum.valueOf(policyType); + } catch (Throwable t) { + return StorServerConfig.Merge_throttling_policy.Type.STATIC; + } + } + StorServerProducer(String clusterName, ModelContext.FeatureFlags featureFlags) { this.clusterName = clusterName; maxMergesPerNode = featureFlags.maxConcurrentMergesPerNode(); queueSize = featureFlags.maxMergeQueueSize(); + mergeThrottlingPolicyType = toThrottlePolicyType(featureFlags.mergeThrottlingPolicy()); } @Override @@ -73,5 +83,7 @@ public class StorServerProducer implements StorServerConfig.Producer { if (bucketDBStripeBits != null) { builder.content_node_bucket_db_stripe_bits(bucketDBStripeBits); } + // TODO set throttle policy params based on existing or separate flags + builder.merge_throttling_policy(new StorServerConfig.Merge_throttling_policy.Builder().type(mergeThrottlingPolicyType)); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 06b907ca2c2..2953b0cb32d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -160,6 +160,25 @@ public class StorageClusterTest { } @Test + public void merge_throttling_policy_config_defaults_to_static() { + var config = configFromProperties(new TestProperties()); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + + @Test + public void merge_throttling_policy_config_is_derived_from_flag() { + var config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("STATIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("DYNAMIC")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.DYNAMIC, config.merge_throttling_policy().type()); + + // Invalid enum values fall back to the default + config = configFromProperties(new TestProperties().setMergeThrottlingPolicy("UKULELE")); + assertEquals(StorServerConfig.Merge_throttling_policy.Type.STATIC, config.merge_throttling_policy().type()); + } + + @Test public void testVisitors() { StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder(); parse(cluster("bees", diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index fddb80610cd..89d3318d710 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -201,6 +201,7 @@ public class ModelContextImpl implements ModelContext { private final List<String> ignoredHttpUserAgents; private final boolean enableServerOcspStapling; private final String persistenceAsyncThrottling; + private final String mergeThrottlingPolicy; private final boolean useQrserverServiceName; public FeatureFlags(FlagSource source, ApplicationId appId) { @@ -241,6 +242,7 @@ public class ModelContextImpl implements ModelContext { this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); + this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); } @@ -283,6 +285,7 @@ public class ModelContextImpl implements ModelContext { @Override public List<String> ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } + @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 30a7c2cd600..470bf9a0c1c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -357,6 +357,14 @@ public class Flags { "Triggers restart, takes effect immediately", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag MERGE_THROTTLING_POLICY = defineStringFlag( + "merge-throttling-policy", "STATIC", + List.of("vekterli"), "2022-01-25", "2022-05-01", + "Sets the policy used for merge throttling on the content nodes. " + + "Valid values: STATIC, DYNAMIC", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag CHECK_CONFIG_CONVERGENCE_BEFORE_RESTARTING = defineFeatureFlag( "check-config-convergence-before-restart", false, List.of("hmusum"), "2022-01-16", "2022-02-16", |