Add feature flag for two-phase document GC in content clusters

6 files changed, 51 insertions, 2 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index e45ab5de253..f439e88f260 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -129,6 +129,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); }
         @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; }
         @ModelFeatureFlag(owners = {"arnej"}) default String logFileCompressionAlgorithm(String defVal) { return defVal; }
+        @ModelFeatureFlag(owners = {"vekterli"}) default boolean useTwoPhaseDocumentGc() { return false; }
     }
 
     /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 8e2f3feb010..c2218cef684 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -79,6 +79,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private boolean enableBitVectors = false;
     private boolean loadCodeAsHugePages = false;
     private boolean sharedStringRepoNoReclaim = false;
+    private boolean useTwoPhaseDocumentGc = false;
     private int mbus_java_num_targets = 1;
     private int mbus_java_events_before_wakeup = 1;
     private int mbus_cpp_num_targets = 1;
@@ -149,6 +150,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public int rpcNumTargets() { return rpc_num_targets; }
     @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; }
     @Override public String queryDispatchPolicy() { return queryDispatchPolicy; }
+    @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; }
 
 
     public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) {
@@ -413,6 +415,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setUseTwoPhaseDocumentGc(boolean useTwoPhase) {
+        this.useTwoPhaseDocumentGc = useTwoPhase;
+        return this;
+    }
+
     public static class Spec implements ConfigServerSpec {
 
         private final String hostName;
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
index b8d2a4f91fe..b8e27d92f8c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
@@ -43,6 +43,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
     private final boolean useThreePhaseUpdates;
     private final int maxActivationInhibitedOutOfSyncGroups;
     private final boolean unorderedMergeChaining;
+    private final boolean useTwoPhaseDocumentGc;
 
     public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> {
 
@@ -106,11 +107,12 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
             boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates();
             int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups();
             boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining();
+            boolean useTwoPhaseDocumentGc = deployState.getProperties().featureFlags().useTwoPhaseDocumentGc();
 
             return new DistributorCluster(parent,
                     new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc,
                     hasIndexedDocumentType, useThreePhaseUpdates,
-                    maxInhibitedGroups, unorderedMergeChaining);
+                    maxInhibitedGroups, unorderedMergeChaining, useTwoPhaseDocumentGc);
         }
     }
 
@@ -118,7 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
                                GcOptions gc, boolean hasIndexedDocumentType,
                                boolean useThreePhaseUpdates,
                                int maxActivationInhibitedOutOfSyncGroups,
-                               boolean unorderedMergeChaining)
+                               boolean unorderedMergeChaining,
+                               boolean useTwoPhaseDocumentGc)
     {
         super(parent, "distributor");
         this.parent = parent;
@@ -128,6 +131,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
         this.useThreePhaseUpdates = useThreePhaseUpdates;
         this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups;
         this.unorderedMergeChaining = unorderedMergeChaining;
+        this.useTwoPhaseDocumentGc = useTwoPhaseDocumentGc;
     }
 
     @Override
@@ -142,6 +146,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl
         builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates);
         builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups);
         builder.use_unordered_merge_chaining(unorderedMergeChaining);
+        builder.enable_two_phase_garbage_collection(useTwoPhaseDocumentGc);
 
         bucketSplitting.getConfig(builder);
     }
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 17ce43bba2c..ac291fc578f 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1165,6 +1165,32 @@ public class ContentClusterTest extends ContentBaseTest {
         return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining();
     }
 
+    private boolean resolveTwoPhaseGcConfigWithFeatureFlag(Boolean flagEnableTwoPhase) {
+        var props = new TestProperties();
+        if (flagEnableTwoPhase != null) {
+            props.setUseTwoPhaseDocumentGc(flagEnableTwoPhase);
+        }
+        VespaModel model = createEnd2EndOneNode(props);
+
+        ContentCluster cc = model.getContentClusters().get("storage");
+        var builder = new StorDistributormanagerConfig.Builder();
+        cc.getDistributorNodes().getConfig(builder);
+
+        return (new StorDistributormanagerConfig(builder)).enable_two_phase_garbage_collection();
+    }
+
+    @Test
+    public void two_phase_garbage_collection_config_is_controlled_by_properties() {
+        assertFalse(resolveTwoPhaseGcConfigWithFeatureFlag(false));
+        assertTrue(resolveTwoPhaseGcConfigWithFeatureFlag(true));
+    }
+
+    // TODO change once gradual rollout complete
+    @Test
+    public void two_phase_garbage_collection_config_is_disabled_by_default() {
+        assertFalse(resolveTwoPhaseGcConfigWithFeatureFlag(null));
+    }
+
     @Test
     void testDedicatedClusterControllers() {
         VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true)
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index 7c7a12bbf36..69551c62840 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -211,6 +211,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean enableProxyProtocolMixedMode;
         private final boolean sharedStringRepoNoReclaim;
         private final String logFileCompressionAlgorithm;
+        private final boolean useTwoPhaseDocumentGc;
         private final boolean mbus_dispatch_on_decode;
         private final boolean mbus_dispatch_on_encode;
         private final int mbus_threads;
@@ -271,6 +272,7 @@ public class ModelContextImpl implements ModelContext {
             this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE);
             this.sharedStringRepoNoReclaim = flagValue(source, appId, version, Flags.SHARED_STRING_REPO_NO_RECLAIM);
             this.logFileCompressionAlgorithm = flagValue(source, appId, version, Flags.LOG_FILE_COMPRESSION_ALGORITHM);
+            this.useTwoPhaseDocumentGc = flagValue(source, appId, version, Flags.USE_TWO_PHASE_DOCUMENT_GC);
             this.mbus_java_num_targets = flagValue(source, appId, version, Flags.MBUS_JAVA_NUM_TARGETS);
             this.mbus_java_events_before_wakeup = flagValue(source, appId, version, Flags.MBUS_JAVA_EVENTS_BEFORE_WAKEUP);
             this.mbus_cpp_num_targets = flagValue(source, appId, version, Flags.MBUS_CPP_NUM_TARGETS);
@@ -343,6 +345,7 @@ public class ModelContextImpl implements ModelContext {
             }
             return defVal;
         }
+        @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index 21f95c7ac6a..5527a565cf4 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -488,6 +488,13 @@ public class Flags {
             "Takes effect next maintenance run"
     );
 
+    public static final UnboundBooleanFlag USE_TWO_PHASE_DOCUMENT_GC = defineFeatureFlag(
+            "use-two-phase-document-gc", false,
+            List.of("vekterli"), "2022-08-24", "2022-11-01",
+            "Use two-phase document GC in content clusters",
+            "Takes effect at redeployment",
+            ZONE_ID, APPLICATION_ID);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,