Merge pull request #20950 from vespa-engine/vekterli/support-live-reconfig-of-dynamic-throttle-params

Add feature flags for dynamic persistence throttling window size tuning [run-systemtest]

3 files changed, 57 insertions, 10 deletions

diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 3aadf30e2f9..a492ee47030 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -72,6 +72,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private boolean failDeploymentWithInvalidJvmOptions = false;
     private String persistenceAsyncThrottling = "UNLIMITED";
     private String mergeThrottlingPolicy = "STATIC";
+    private double persistenceThrottlingWsDecrementFactor = 1.2;
+    private double persistenceThrottlingWsBackoff = 0.95;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -123,6 +125,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; }
     @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; }
     @Override public String mergeThrottlingPolicy() { return mergeThrottlingPolicy; }
+    @Override public double persistenceThrottlingWsDecrementFactor() { return persistenceThrottlingWsDecrementFactor; }
+    @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; }
 
     public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) {
         this.maxUnCommittedMemory = maxUnCommittedMemory;
@@ -320,6 +324,16 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setPersistenceThrottlingWsDecrementFactor(double factor) {
+        this.persistenceThrottlingWsDecrementFactor = factor;
+        return this;
+    }
+
+    public TestProperties setPersistenceThrottlingWsBackoff(double backoff) {
+        this.persistenceThrottlingWsBackoff = backoff;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
index 5be06306f89..f3263658717 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
@@ -46,7 +46,9 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
     private final ContentCluster cluster;
     private final int reponseNumThreads;
     private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType;
-    private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType;
+    private final StorFilestorConfig.Async_operation_throttler.Type.Enum asyncOperationThrottlerType;
+    private final double persistenceThrottlingWsDecrementFactor;
+    private final double persistenceThrottlingWsBackoff;
     private final boolean useAsyncMessageHandlingOnSchedule;
 
     private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) {
@@ -57,11 +59,11 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
         }
     }
 
-    private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) {
+    private static StorFilestorConfig.Async_operation_throttler.Type.Enum toAsyncOperationThrottlerType(String throttlerType) {
         try {
-            return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType);
+            return StorFilestorConfig.Async_operation_throttler.Type.Enum.valueOf(throttlerType);
         } catch (Throwable t) {
-            return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED;
+            return StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED;
         }
     }
 
@@ -71,7 +73,9 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
         this.reponseNumThreads = featureFlags.defaultNumResponseThreads();
         this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType());
         this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling());
-        useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule();
+        this.persistenceThrottlingWsDecrementFactor = featureFlags.persistenceThrottlingWsDecrementFactor();
+        this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff();
+        this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule();
     }
 
     @Override
@@ -83,7 +87,16 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
         builder.num_response_threads(reponseNumThreads);
         builder.response_sequencer_type(responseSequencerType);
         builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule);
-        builder.async_operation_throttler_type(asyncOperationThrottlerType);
+        // TODO remove deprecated throttler type config
+        builder.async_operation_throttler_type((asyncOperationThrottlerType == StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC)
+                                               ? StorFilestorConfig.Async_operation_throttler_type.Enum.DYNAMIC
+                                               : StorFilestorConfig.Async_operation_throttler_type.Enum.UNLIMITED);
+
+        var throttleBuilder = new StorFilestorConfig.Async_operation_throttler.Builder();
+        throttleBuilder.type(asyncOperationThrottlerType);
+        throttleBuilder.window_size_decrement_factor(persistenceThrottlingWsDecrementFactor);
+        throttleBuilder.window_size_backoff(persistenceThrottlingWsBackoff);
+        builder.async_operation_throttler(throttleBuilder);
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 2953b0cb32d..509055df7f4 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -309,20 +309,40 @@ public class StorageClusterTest {
     @Test
     public void persistence_async_throttle_config_defaults_to_unlimited() {
         var config = filestorConfigFromProducer(simpleCluster(new TestProperties()));
-        assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type());
+        assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove
+        assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type());
     }
 
     @Test
     public void persistence_async_throttle_config_is_derived_from_flag() {
         var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED")));
-        assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type());
+        assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove
+        assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type());
 
         config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC")));
-        assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type());
+        assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); // TODO remove
+        assertEquals(StorFilestorConfig.Async_operation_throttler.Type.DYNAMIC, config.async_operation_throttler().type());
 
         // Invalid enum values fall back to the default
         config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS")));
-        assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type());
+        assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); // TODO remove
+        assertEquals(StorFilestorConfig.Async_operation_throttler.Type.UNLIMITED, config.async_operation_throttler().type());
+    }
+
+    @Test
+    public void persistence_dynamic_throttling_parameters_have_sane_defaults() {
+        var config = filestorConfigFromProducer(simpleCluster(new TestProperties()));
+        assertEquals(1.2, config.async_operation_throttler().window_size_decrement_factor(), 0.0001);
+        assertEquals(0.95, config.async_operation_throttler().window_size_backoff(), 0.0001);
+    }
+
+    @Test
+    public void persistence_dynamic_throttling_parameters_can_be_set_through_feature_flags() {
+        var config = filestorConfigFromProducer(simpleCluster(new TestProperties()
+                .setPersistenceThrottlingWsDecrementFactor(1.5)
+                .setPersistenceThrottlingWsBackoff(0.8)));
+        assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001);
+        assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001);
     }
 
     @Test