Add feature flag for controlling throttling of merge feed ops

6 files changed, 28 insertions, 1 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 37b32cdd1cc..e0e4318ccc8 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -111,6 +111,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsBackoff() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"vekterli"}) default int persistenceThrottlingWindowSize() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"vekterli"}) default double persistenceThrottlingWsResizeRate() { throw new UnsupportedOperationException("TODO specify default value"); }
+        @ModelFeatureFlag(owners = {"vekterli"}) default boolean persistenceThrottlingOfMergeFeedOps() { throw new UnsupportedOperationException("TODO specify default value"); }
         @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean inhibitDefaultMergesWhenGlobalMergesPending() { return false; }
         @ModelFeatureFlag(owners = {"arnej"}) default boolean useQrserverServiceName() { return true; }
         @ModelFeatureFlag(owners = {"bjorncs", "baldersheim"}) default boolean enableJdiscPreshutdownCommand() { return true; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 41fdea27ee6..51f3455762c 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -74,6 +74,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private double persistenceThrottlingWsBackoff = 0.95;
     private int persistenceThrottlingWindowSize = -1;
     private double persistenceThrottlingWsResizeRate = 3.0;
+    private boolean persistenceThrottlingOfMergeFeedOps = true;
     private boolean inhibitDefaultMergesWhenGlobalMergesPending = false;
     private boolean useV8GeoPositions = false;
     private List<String> environmentVariables = List.of();
@@ -132,6 +133,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; }
     @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; }
     @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; }
+    @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; }
     @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; }
     @Override public boolean useV8GeoPositions() { return useV8GeoPositions; }
     @Override public List<String> environmentVariables() { return environmentVariables; }
@@ -348,6 +350,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setPersistenceThrottlingOfMergeFeedOps(boolean throttleOps) {
+        this.persistenceThrottlingOfMergeFeedOps = throttleOps;
+        return this;
+    }
+
     public TestProperties inhibitDefaultMergesWhenGlobalMergesPending(boolean value) {
         this.inhibitDefaultMergesWhenGlobalMergesPending = value;
         return this;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
index 81a43108157..a1c4fc41d55 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
@@ -51,6 +51,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
     private final double persistenceThrottlingWsBackoff;
     private final int persistenceThrottingWindowSize;
     private final double persistenceThrottlingWsResizeRate;
+    private final boolean persistenceThrottlingOfMergeFeedOps;
     private final boolean useAsyncMessageHandlingOnSchedule;
 
     private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) {
@@ -79,6 +80,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
         this.persistenceThrottlingWsBackoff = featureFlags.persistenceThrottlingWsBackoff();
         this.persistenceThrottingWindowSize = featureFlags.persistenceThrottlingWindowSize();
         this.persistenceThrottlingWsResizeRate = featureFlags.persistenceThrottlingWsResizeRate();
+        this.persistenceThrottlingOfMergeFeedOps = featureFlags.persistenceThrottlingOfMergeFeedOps();
         this.useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule();
     }
 
@@ -105,6 +107,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
             throttleBuilder.max_window_size(persistenceThrottingWindowSize);
         }
         throttleBuilder.resize_rate(persistenceThrottlingWsResizeRate);
+        throttleBuilder.throttle_individual_merge_feed_ops(persistenceThrottlingOfMergeFeedOps);
         builder.async_operation_throttler(throttleBuilder);
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index fd4a21a0dd0..ebc94e1f481 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -337,6 +337,7 @@ public class StorageClusterTest {
         assertEquals(20, config.async_operation_throttler().min_window_size());
         assertEquals(-1, config.async_operation_throttler().max_window_size()); // <=0 implies +inf
         assertEquals(3.0, config.async_operation_throttler().resize_rate(), 0.0001);
+        assertTrue(config.async_operation_throttler().throttle_individual_merge_feed_ops());
     }
 
     @Test
@@ -345,13 +346,15 @@ public class StorageClusterTest {
                 .setPersistenceThrottlingWsDecrementFactor(1.5)
                 .setPersistenceThrottlingWsBackoff(0.8)
                 .setPersistenceThrottlingWindowSize(42)
-                .setPersistenceThrottlingWsResizeRate(2.5)));
+                .setPersistenceThrottlingWsResizeRate(2.5)
+                .setPersistenceThrottlingOfMergeFeedOps(false)));
         assertEquals(1.5, config.async_operation_throttler().window_size_decrement_factor(), 0.0001);
         assertEquals(0.8, config.async_operation_throttler().window_size_backoff(), 0.0001);
         // If window size is set, min and max are locked to the same value
         assertEquals(42, config.async_operation_throttler().min_window_size());
         assertEquals(42, config.async_operation_throttler().max_window_size());
         assertEquals(2.5, config.async_operation_throttler().resize_rate(), 0.0001);
+        assertFalse(config.async_operation_throttler().throttle_individual_merge_feed_ops());
     }
 
     @Test
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 9cd4b37133f..5a813c7886a 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext {
         private final double persistenceThrottlingWsBackoff;
         private final int persistenceThrottlingWindowSize;
         private final double persistenceThrottlingWsResizeRate;
+        private final boolean persistenceThrottlingOfMergeFeedOps;
         private final boolean inhibitDefaultMergesWhenGlobalMergesPending;
         private final boolean useQrserverServiceName;
         private final boolean avoidRenamingSummaryFeatures;
@@ -251,6 +252,7 @@ public class ModelContextImpl implements ModelContext {
             this.persistenceThrottlingWsBackoff = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_BACKOFF);
             this.persistenceThrottlingWindowSize = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WINDOW_SIZE);
             this.persistenceThrottlingWsResizeRate = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_WS_RESIZE_RATE);
+            this.persistenceThrottlingOfMergeFeedOps = flagValue(source, appId, Flags.PERSISTENCE_THROTTLING_OF_MERGE_FEED_OPS);
             this.inhibitDefaultMergesWhenGlobalMergesPending = flagValue(source, appId, Flags.INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING);
             this.useQrserverServiceName =  flagValue(source, appId, Flags.USE_QRSERVER_SERVICE_NAME);
             this.avoidRenamingSummaryFeatures = flagValue(source, appId, Flags.AVOID_RENAMING_SUMMARY_FEATURES);
@@ -299,6 +301,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public double persistenceThrottlingWsBackoff() { return persistenceThrottlingWsBackoff; }
         @Override public int persistenceThrottlingWindowSize() { return persistenceThrottlingWindowSize; }
         @Override public double persistenceThrottlingWsResizeRate() { return persistenceThrottlingWsResizeRate; }
+        @Override public boolean persistenceThrottlingOfMergeFeedOps() { return persistenceThrottlingOfMergeFeedOps; }
         @Override public boolean inhibitDefaultMergesWhenGlobalMergesPending() { return inhibitDefaultMergesWhenGlobalMergesPending; }
         @Override public boolean useQrserverServiceName() { return useQrserverServiceName; }
         @Override public boolean avoidRenamingSummaryFeatures() { return avoidRenamingSummaryFeatures; }
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index a9c0711f148..271c75fe446 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -344,6 +344,16 @@ public class Flags {
             "Takes effect on redeployment",
             ZONE_ID, APPLICATION_ID);
 
+    public static final UnboundBooleanFlag PERSISTENCE_THROTTLING_OF_MERGE_FEED_OPS = defineFeatureFlag(
+            "persistence-throttling-of-merge-feed-ops", true,
+            List.of("vekterli"), "2022-02-24", "2022-06-01",
+            "If true, each put/remove contained within a merge is individually throttled as if it " +
+            "were a put/remove from a client. If false, merges are throttled at a persistence thread " +
+            "level, i.e. per ApplyBucketDiff message, regardless of how many document operations " +
+            "are contained within. Only applies if DYNAMIC policy is used.",
+            "Takes effect on redeployment",
+            ZONE_ID, APPLICATION_ID);
+
     public static final UnboundBooleanFlag INHIBIT_DEFAULT_MERGES_WHEN_GLOBAL_MERGES_PENDING = defineFeatureFlag(
             "inhibit-default-merges-when-global-merges-pending", false,
             List.of("geirst", "vekterli"), "2022-02-11", "2022-06-01",