Merge pull request #19702 from vespa-engine/geirst/async-apply-bucket-diff-feature-flag

Add feature flag for "async-apply-bucket-diff".

6 files changed, 49 insertions, 27 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index 2c51aa89c66..8c2957502a8 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -103,6 +103,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default int distributorMergeBusyWait() { return 10; }
         @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean distributorEnhancedMaintenanceScheduling() { return false; }
         @ModelFeatureFlag(owners = {"arnej"}) default boolean forwardIssuesAsErrors() { return true; }
+        @ModelFeatureFlag(owners = {"geirst", "vekterli"}) default boolean asyncApplyBucketDiff() { return false; }
     }
 
     /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 7a09a36c1d7..b7506587102 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -70,6 +70,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private int maxUnCommittedMemory = 123456;
     private double diskBloatFactor = 0.2;
     private boolean distributorEnhancedMaintenanceScheduling = false;
+    private boolean asyncApplyBucketDiff = false;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -120,6 +121,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; }
     @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; }
     @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; }
+    @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; }
 
     public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) {
         this.maxUnCommittedMemory = maxUnCommittedMemory;
@@ -307,6 +309,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setAsyncApplyBucketDiff(boolean value) {
+        asyncApplyBucketDiff = value;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
index 5c692e8ef6b..d7f6fb6c581 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
@@ -47,6 +47,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
     private final int reponseNumThreads;
     private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType;
     private final boolean useAsyncMessageHandlingOnSchedule;
+    private final boolean asyncApplyBucketDiff;
 
     private static StorFilestorConfig.Response_sequencer_type.Enum convertResponseSequencerType(String sequencerType) {
         try {
@@ -62,6 +63,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
         this.reponseNumThreads = featureFlags.defaultNumResponseThreads();
         this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType());
         useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule();
+        asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff();
     }
 
     @Override
@@ -73,6 +75,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
         builder.num_response_threads(reponseNumThreads);
         builder.response_sequencer_type(responseSequencerType);
         builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule);
+        builder.async_apply_bucket_diff(asyncApplyBucketDiff);
     }
 
 }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index 9d8d7509966..739f8b7fff2 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -142,6 +142,16 @@ public class StorageClusterTest {
         return new StorServerConfig(builder);
     }
 
+    private StorFilestorConfig filestorConfigFromProducer(StorFilestorConfig.Producer producer) {
+        var builder = new StorFilestorConfig.Builder();
+        producer.getConfig(builder);
+        return new StorFilestorConfig(builder);
+    }
+
+    private StorFilestorConfig filestorConfigFromProperties(TestProperties properties) {
+        return filestorConfigFromProducer(parse(cluster("foo", ""), properties));
+    }
+
     @Test
     public void testMergeFeatureFlags() {
         var config = configFromProperties(new TestProperties().setMaxMergeQueueSize(1919).setMaxConcurrentMergesPerNode(37));
@@ -159,6 +169,15 @@ public class StorageClusterTest {
     }
 
     @Test
+    public void async_apply_bucket_diff_can_be_controlled_by_feature_flag() {
+        var config = filestorConfigFromProperties(new TestProperties());
+        assertFalse(config.async_apply_bucket_diff());
+
+        config = filestorConfigFromProperties(new TestProperties().setAsyncApplyBucketDiff(true));
+        assertTrue(config.async_apply_bucket_diff());
+    }
+
+    @Test
     public void testVisitors() {
         StorVisitorConfig.Builder builder = new StorVisitorConfig.Builder();
         parse(cluster("bees",
@@ -188,9 +207,7 @@ public class StorageClusterTest {
         );
 
         {
-            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-            stc.getConfig(builder);
-            StorFilestorConfig config = new StorFilestorConfig(builder);
+            var config = filestorConfigFromProducer(stc);
 
             assertEquals(7, config.num_threads());
             assertFalse(config.enable_multibit_split_optimalization());
@@ -199,9 +216,7 @@ public class StorageClusterTest {
         {
             assertEquals(1, stc.getChildren().size());
             StorageNode sn = stc.getChildren().values().iterator().next();
-            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-            sn.getConfig(builder);
-            StorFilestorConfig config = new StorFilestorConfig(builder);
+            var config = filestorConfigFromProducer(sn);
             assertEquals(7, config.num_threads());
         }
     }
@@ -215,9 +230,7 @@ public class StorageClusterTest {
                 "</tuning>")),
                 new Flavor(new FlavorsConfig.Flavor.Builder().name("test-flavor").minCpuCores(9).build())
         );
-        StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-        stc.getConfig(builder);
-        StorFilestorConfig config = new StorFilestorConfig(builder);
+        var config = filestorConfigFromProducer(stc);
         assertEquals(2, config.num_response_threads());
         assertEquals(StorFilestorConfig.Response_sequencer_type.ADAPTIVE, config.response_sequencer_type());
         assertEquals(7, config.num_threads());
@@ -238,9 +251,7 @@ public class StorageClusterTest {
         );
 
         {
-            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-            stc.getConfig(builder);
-            StorFilestorConfig config = new StorFilestorConfig(builder);
+            var config = filestorConfigFromProducer(stc);
 
             assertEquals(4, config.num_threads());
             assertFalse(config.enable_multibit_split_optimalization());
@@ -248,9 +259,7 @@ public class StorageClusterTest {
         {
             assertEquals(1, stc.getChildren().size());
             StorageNode sn = stc.getChildren().values().iterator().next();
-            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-            sn.getConfig(builder);
-            StorFilestorConfig config = new StorFilestorConfig(builder);
+            var config = filestorConfigFromProducer(sn);
             assertEquals(4, config.num_threads());
         }
     }
@@ -262,17 +271,13 @@ public class StorageClusterTest {
         );
 
         {
-            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-            stc.getConfig(builder);
-            StorFilestorConfig config = new StorFilestorConfig(builder);
+            var config = filestorConfigFromProducer(stc);
             assertEquals(8, config.num_threads());
         }
         {
             assertEquals(1, stc.getChildren().size());
             StorageNode sn = stc.getChildren().values().iterator().next();
-            StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-            sn.getConfig(builder);
-            StorFilestorConfig config = new StorFilestorConfig(builder);
+            var config = filestorConfigFromProducer(sn);
             assertEquals(9, config.num_threads());
         }
     }
@@ -285,17 +290,13 @@ public class StorageClusterTest {
 
     @Test
     public void testFeatureFlagControlOfResponseSequencer() {
-        StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-        simpleCluster(new TestProperties().setResponseNumThreads(13).setResponseSequencerType("THROUGHPUT")).getConfig(builder);
-        StorFilestorConfig config = new StorFilestorConfig(builder);
+        var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setResponseNumThreads(13).setResponseSequencerType("THROUGHPUT")));
         assertEquals(13, config.num_response_threads());
         assertEquals(StorFilestorConfig.Response_sequencer_type.THROUGHPUT, config.response_sequencer_type());
     }
 
     private void verifyAsyncMessageHandlingOnSchedule(boolean expected, boolean value) {
-        StorFilestorConfig.Builder builder = new StorFilestorConfig.Builder();
-        simpleCluster(new TestProperties().setAsyncMessageHandlingOnSchedule(value)).getConfig(builder);
-        StorFilestorConfig config = new StorFilestorConfig(builder);
+        var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setAsyncMessageHandlingOnSchedule(value)));
         assertEquals(expected, config.use_async_message_handling_on_schedule());
     }
     @Test
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index b2360ef262d..7fdc18827f4 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -197,6 +197,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean distributorEnhancedMaintenanceScheduling;
         private final int maxUnCommittedMemory;
         private final boolean forwardIssuesAsErrors;
+        private final boolean asyncApplyBucketDiff;
 
         public FeatureFlags(FlagSource source, ApplicationId appId) {
             this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -232,6 +233,7 @@ public class ModelContextImpl implements ModelContext {
             this.distributorEnhancedMaintenanceScheduling = flagValue(source, appId, Flags.DISTRIBUTOR_ENHANCED_MAINTENANCE_SCHEDULING);
             this.maxUnCommittedMemory = flagValue(source, appId, Flags.MAX_UNCOMMITTED_MEMORY);;
             this.forwardIssuesAsErrors = flagValue(source, appId, PermanentFlags.FORWARD_ISSUES_AS_ERRORS);
+            this.asyncApplyBucketDiff = flagValue(source, appId, Flags.ASYNC_APPLY_BUCKET_DIFF);
         }
 
         @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; }
@@ -269,6 +271,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; }
         @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; }
         @Override public boolean forwardIssuesAsErrors() { return forwardIssuesAsErrors; }
+        @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index e355bb4d6c4..4b7d05928c0 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -369,6 +369,13 @@ public class Flags {
             "Takes effect at redeploy",
             ZONE_ID, APPLICATION_ID);
 
+    public static final UnboundBooleanFlag ASYNC_APPLY_BUCKET_DIFF = defineFeatureFlag(
+            "async-apply-bucket-diff", false,
+            List.of("geirst", "vekterli"), "2021-10-22", "2022-01-31",
+            "Whether portions of apply bucket diff handling will be performed asynchronously",
+            "Takes effect at redeploy",
+            ZONE_ID, APPLICATION_ID);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,