From 5e425b8215b05e573ee9321f1cae509bb259d370 Mon Sep 17 00:00:00 2001 From: Tor Brede Vekterli Date: Wed, 12 Jan 2022 14:10:53 +0100 Subject: Add feature flag for controlling throttling of async persistence operations Currently defaults to `UNLIMITED` throttling policy, i.e. legacy behavior. --- .../java/com/yahoo/config/model/api/ModelContext.java | 1 + .../com/yahoo/config/model/deploy/TestProperties.java | 7 +++++++ .../content/storagecluster/FileStorProducer.java | 11 +++++++++++ .../yahoo/vespa/model/content/StorageClusterTest.java | 19 +++++++++++++++++++ .../vespa/config/server/deploy/ModelContextImpl.java | 3 +++ flags/src/main/java/com/yahoo/vespa/flags/Flags.java | 8 ++++++++ 6 files changed, 49 insertions(+) diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 2ea1ef186d2..42718658fbf 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -114,6 +114,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default double tlsSizeFraction() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"arnej", "andreer"}) default List ignoredHttpUserAgents() { return List.of(); } @ModelFeatureFlag(owners = {"bjorncs"}) default boolean enableServerOcspStapling() { return false; } + @ModelFeatureFlag(owners = {"vekterli"}) default String persistenceAsyncThrottling() { throw new UnsupportedOperationException("TODO specify default value"); } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 0007fa0d18a..2dc91ab1b1f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -77,6 +77,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; private double tlsSizeFraction = 0.02; + private String persistenceAsyncThrottling = "UNLIMITED"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -134,6 +135,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } @Override public double tlsSizeFraction() { return tlsSizeFraction; } + @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -356,6 +358,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setPersistenceAsyncThrottling(String type) { + this.persistenceAsyncThrottling = type; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index d7f6fb6c581..1f3a76b766e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,6 +46,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; + private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; private final boolean useAsyncMessageHandlingOnSchedule; private final boolean asyncApplyBucketDiff; @@ -57,11 +58,20 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } + private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) { + try { + return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType); + } catch (Throwable t) { + return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED; + } + } + public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); + this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff(); } @@ -76,6 +86,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); builder.async_apply_bucket_diff(asyncApplyBucketDiff); + builder.async_operation_throttler_type(asyncOperationThrottlerType); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index 739f8b7fff2..740193feeff 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -305,6 +305,25 @@ public class StorageClusterTest { verifyAsyncMessageHandlingOnSchedule(true, true); } + @Test + public void persistence_async_throttle_config_defaults_to_unlimited() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + } + + @Test + public void persistence_async_throttle_config_is_derived_from_flag() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + + config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); + + // Invalid enum values fall back to the default + config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + } + @Test public void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() { StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7d4ac948194..b32c82399f6 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -206,6 +206,7 @@ public class ModelContextImpl implements ModelContext { private final double tlsSizeFraction; private final List ignoredHttpUserAgents; private final boolean enableServerOcspStapling; + private final String persistenceAsyncThrottling; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -250,6 +251,7 @@ public class ModelContextImpl implements ModelContext { this.tlsSizeFraction = flagValue(source, appId, Flags.TLS_SIZE_FRACTION); this.ignoredHttpUserAgents = flagValue(source, appId, PermanentFlags.IGNORED_HTTP_USER_AGENTS); this.enableServerOcspStapling = flagValue(source, appId, Flags.ENABLE_SERVER_OCSP_STAPLING); + this.persistenceAsyncThrottling = flagValue(source, appId, Flags.PERSISTENCE_ASYNC_THROTTLING); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -296,6 +298,7 @@ public class ModelContextImpl implements ModelContext { @Override public double tlsSizeFraction() { return tlsSizeFraction; } @Override public List ignoredHttpUserAgents() { return ignoredHttpUserAgents; } @Override public boolean enableServerOcspStapling() { return enableServerOcspStapling; } + @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } private static V flagValue(FlagSource source, ApplicationId appId, UnboundFlag flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 2a36c62239a..5dc418e6e6c 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -408,6 +408,14 @@ public class Flags { "Takes effect on Docker container restart", ZONE_ID, APPLICATION_ID, NODE_TYPE); + public static final UnboundStringFlag PERSISTENCE_ASYNC_THROTTLING = defineStringFlag( + "persistence-async-throttling", "UNLIMITED", + List.of("vekterli"), "2022-01-12", "2022-05-01", + "Sets the throttling policy used for async persistence operations on the content nodes. " + + "Valid values: UNLIMITED, DYNAMIC", + "Triggers restart, takes effect immediately", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List owners, String createdAt, String expiresAt, String description, -- cgit v1.2.3