diff options
author | Tor Brede Vekterli <vekterli@yahooinc.com> | 2022-02-23 16:51:53 +0100 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@yahooinc.com> | 2022-02-23 16:51:53 +0100 |
commit | c4cb61ae2b5b32be6d157a866d132f86e7087ea0 (patch) | |
tree | 0634f10406e5b9d1ef3f198e250ee50d96bcbf8f | |
parent | d015e7a9ef5f6e528fa8bb74949d46c4e9b2dcbc (diff) |
Add more feature flags for tuning dynamic persistence throttling
Adds the following feature flags:
* `persistence-throttling-window-size` for setting min/max window size
to the same value, effectively creating a static throttle policy.
* `persistence-throttling-ws-resize-rate` for controlling the rate
at which the dynamic throttle policy resizes its window.
6 files changed, 57 insertions, 1 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index b4ece2dc28b..37b32cdd1cc 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -109,6 +109,8 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default String mergeThrottlingPolicy() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsDecrementFactor() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default int persistenceThrottlingWindowSize() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsResizeRate() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean inhibitDefaultMergesWhenGlobalMergesPending() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; } @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9ae1a75b858..41fdea27ee6 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -72,6 +72,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String mergeThrottlingPolicy = "STATIC"; private double persistenceThrottlingWsDecrementFactor = 1.2; private double persistenceThrottlingWsBackoff = 0.95; + private int persistenceThrottlingWindowSize = -1; + private double persistenceThrottlingWsResizeRate = 3.0; private boolean inhibitDefaultMergesWhenGlobalMergesPending = false; private boolean useV8GeoPositions = false; private List<String> environmentVariables = List.of(); @@ -128,6 +130,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; } + @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; } @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useV8GeoPositions() { return useV8GeoPositions; } @Override public List<String> environmentVariables() { return environmentVariables; } @@ -334,6 +338,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setPersistenceThrottlingWindowSize(int windowSize) { + this.persistenceThrottlingWindowSize = windowSize; + return this; + } + + public TestProperties setPersistenceThrottlingWsResizeRate(double resizeRate) { + this.persistenceThrottlingWsResizeRate = resizeRate; + return this; + } + public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) { this.inhibitDefaultMergesWhenGlobalMergesPending = value; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index f3263658717..81a43108157 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -49,6 +49,8 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; + private final int persistenceThrottingWindowSize; + private final double persistenceThrottlingWsResizeRate; private final boolean useAsyncMessageHandlingOnSchedule; private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) { @@ -75,6 +77,8 @@ public class FileStorProducer implements StorFilestorConfig.Producer { this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor(); this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff(); + this.persistenceThrottingWindowSize = featureFlags.persistenceThrottlingWindowSize(); + this.persistenceThrottlingWsResizeRate = featureFlags.persistenceThrottlingWsResizeRate(); this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); } @@ -96,6 +100,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer { throttleBuilder.type(asyncOperationThrottlerType); throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor); throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff); + if (persistenceThrottingWindowSize > 0) { + throttleBuilder.min_window_size(persistenceThrottingWindowSize); + throttleBuilder.max_window_size(persistenceThrottingWindowSize); + } + throttleBuilder.resize_rate(persistenceThrottlingWsResizeRate); builder.async_operation_throttler(throttleBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 509055df7f4..fd4a21a0dd0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -334,15 +334,24 @@ public class StorageClusterTest { var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001); + assertEquals(20, config.async_operation_throttler().min_window_size()); + assertEquals(-1, config.async_operation_throttler().max_window_size()); // <=0 implies +inf + assertEquals(3.0, config.async_operation_throttler().resize_rate(), 0.0001); } @Test public void persistence_dynamic_throttling_parameters_can_be_set_through_feature_flags() { var config = filestorConfigFromProducer(simpleCluster(new TestProperties() .setPersistenceThrottlingWsDecrementFactor(1.5) - .setPersistenceThrottlingWsBackoff(0.8))); + .setPersistenceThrottlingWsBackoff(0.8) + .setPersistenceThrottlingWindowSize(42) + .setPersistenceThrottlingWsResizeRate(2.5))); assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001); assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001); + // If window size is set, min and max are locked to the same value + assertEquals(42, config.async_operation_throttler().min_window_size()); + assertEquals(42, config.async_operation_throttler().max_window_size()); + assertEquals(2.5, config.async_operation_throttler().resize_rate(), 0.0001); } @Test diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 38245f26a18..9cd4b37133f 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -203,6 +203,8 @@ public class ModelContextImpl implements ModelContext { private final String mergeThrottlingPolicy; private final double persistenceThrottlingWsDecrementFactor; private final double persistenceThrottlingWsBackoff; + private final int persistenceThrottlingWindowSize; + private final double persistenceThrottlingWsResizeRate; private final boolean inhibitDefaultMergesWhenGlobalMergesPending; private final boolean useQrserverServiceName; private final boolean avoidRenamingSummaryFeatures; @@ -247,6 +249,8 @@ public class ModelContextImpl implements ModelContext { this.mergeThrottlingPolicy = flagValue(source, appId, Flags.MERGE_THROTTLING_POLICY); this.persistenceThrottlingWsDecrementFactor = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_DECREMENT_FACTOR); this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF); + this.persistenceThrottlingWindowSize = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WINDOW_SIZE); + this.persistenceThrottlingWsResizeRate = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_RESIZE_RATE); this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING); this.useQrserverServiceName = flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME); this.avoidRenamingSummaryFeatures = flagValue(source, appId, Flags.AVOID_RENAMING_SUMMARY_FEATURES); @@ -293,6 +297,8 @@ public class ModelContextImpl implements ModelContext { @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; } @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; } @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; } + @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; } + @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; } @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; } @Override public boolean useQrserverServiceName() { return useQrserverServiceName; } @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 7aeaf85a80b..a9c0711f148 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -328,6 +328,22 @@ public class Flags { "Takes effect on redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag PERSISTENCE_THROTTLING_WINDOW_SIZE = defineIntFlag( + "persistence-throttling-window-size", -1, + List.of("vekterli"), "2022-02-23", "2022-06-01", + "If greater than zero, sets both min and max window size to the given number, effectively " + + "turning dynamic throttling into a static throttling policy. " + + "Only applies if DYNAMIC policy is used.", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundDoubleFlag PERSISTENCE_THROTTLING_WS_RESIZE_RATE = defineDoubleFlag( + "persistence-throttling-ws-resize-rate", 3.0, + List.of("vekterli"), "2022-02-23", "2022-06-01", + "Sets the dynamic throttle policy resize rate. Only applies if DYNAMIC policy is used.", + "Takes effect on redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundBooleanFlag INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING = defineFeatureFlag( "inhibit-default-merges-when-global-merges-pending", false, List.of("geirst", "vekterli"), "2022-02-11", "2022-06-01", |