Merge pull request #27067 from vespa-engine/vekterli/add-conditional-put-remove-write-repair-feature-flag

Add feature flag for enabling conditional put/remove write repair

6 files changed, 39 insertions, 2 deletions

diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
index ee7ca64166d..41ee47513a0 100644
--- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
+++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java
@@ -115,6 +115,7 @@ public interface ModelContext {
         @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; }
         @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; }
         @ModelFeatureFlag(owners = {"hmusum"}) default boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return false; }
+        @ModelFeatureFlag(owners = {"vekterli", "havardpe"}) default boolean enableConditionalPutRemoveWriteRepair() { return false; }
 
         //Below are all flags that must be kept until 7 is out of the door
         @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; }
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 540905bef4c..56f999f85b4 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -84,6 +84,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     private Optional<CloudAccount> cloudAccount = Optional.empty();
     private boolean allowUserFilters = true;
     private boolean allowMoreThanOneContentGroupDown = false;
+    private boolean enableConditionalPutRemoveWriteRepair = false;
 
     @Override public ModelContext.FeatureFlags featureFlags() { return this; }
     @Override public boolean multitenant() { return multitenant; }
@@ -142,6 +143,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
     @Override public boolean allowUserFilters() { return allowUserFilters; }
     @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only
     @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown; }
+    @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; }
 
     public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) {
         this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim;
@@ -375,6 +377,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
         return this;
     }
 
+    public TestProperties setEnableConditionalPutRemoveWriteRepair(boolean enable) {
+        this.enableConditionalPutRemoveWriteRepair = enable;
+        return this;
+    }
+
     public TestProperties setAllowUserFilters(boolean b) { this.allowUserFilters = b; return this; }
 
     public static class Spec implements ConfigServerSpec {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
index d73f1618dbc..4aac8bfb647 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java
@@ -33,6 +33,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
     private final GcOptions gc;
     private final boolean hasIndexedDocumentType;
     private final int maxActivationInhibitedOutOfSyncGroups;
+    private final boolean enableConditionalPutRemoveWriteRepair;
     public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> {
 
         ContentCluster parent;
@@ -93,17 +94,20 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
             final GcOptions gc = parseGcOptions(documentsNode);
             final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode);
             int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups();
+            boolean enableConditionalPutRemoveWriteRepair = deployState.getProperties().featureFlags().enableConditionalPutRemoveWriteRepair();
 
             return new DistributorCluster(parent,
                     new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc,
                     hasIndexedDocumentType,
-                    maxInhibitedGroups);
+                    maxInhibitedGroups,
+                    enableConditionalPutRemoveWriteRepair);
         }
     }
 
     private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting,
                                GcOptions gc, boolean hasIndexedDocumentType,
-                               int maxActivationInhibitedOutOfSyncGroups)
+                               int maxActivationInhibitedOutOfSyncGroups,
+                               boolean enableConditionalPutRemoveWriteRepair)
     {
         super(parent, "distributor");
         this.parent = parent;
@@ -111,6 +115,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
         this.gc = gc;
         this.hasIndexedDocumentType = hasIndexedDocumentType;
         this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups;
+        this.enableConditionalPutRemoveWriteRepair = enableConditionalPutRemoveWriteRepair;
     }
 
     @Override
@@ -123,6 +128,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen
         builder.enable_revert(parent.getPersistence().supportRevert());
         builder.disable_bucket_activation(!hasIndexedDocumentType);
         builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups);
+        builder.enable_condition_probing(enableConditionalPutRemoveWriteRepair);
         bucketSplitting.getConfig(builder);
     }
 
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 71dd3223da2..14fe7bbcc36 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -1240,6 +1240,17 @@ public class ContentClusterTest extends ContentBaseTest {
         assertEquals(2, resolveMaxInhibitedGroupsConfigWithFeatureFlag(2));
     }
 
+    private boolean resolveConditionProbingFromWriteRepairFeatureFlag(boolean enable) throws Exception {
+        var cfg = resolveStorDistributormanagerConfig(new TestProperties().setEnableConditionalPutRemoveWriteRepair(enable));
+        return cfg.enable_condition_probing();
+    }
+
+    @Test
+    void distributor_condition_probing_is_controlled_by_write_repair_feature_flag() throws Exception {
+        assertFalse(resolveConditionProbingFromWriteRepairFeatureFlag(false));
+        assertTrue(resolveConditionProbingFromWriteRepairFeatureFlag(true));
+    }
+
     private int resolveNumDistributorStripesConfig(Optional<Flavor> flavor) throws Exception {
         var cc = createOneNodeCluster(new TestProperties(), flavor);
         var builder = new StorDistributormanagerConfig.Builder();
diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
index b443a0e64b0..59ddb8b68ab 100644
--- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
+++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java
@@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext {
         private final boolean enableGlobalPhase;
         private final String summaryDecodePolicy;
         private final Predicate<ClusterSpec.Id> allowMoreThanOneContentGroupDown;
+        private final boolean enableConditionalPutRemoveWriteRepair;
 
         public FeatureFlags(FlagSource source, ApplicationId appId, Version version) {
             this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT);
@@ -252,6 +253,7 @@ public class ModelContextImpl implements ModelContext {
             this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE);
             this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY);
             this.allowMoreThanOneContentGroupDown = clusterId -> flagValue(source, appId, version, clusterId, Flags.ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN);
+            this.enableConditionalPutRemoveWriteRepair = flagValue(source, appId, version, Flags.ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR);
         }
 
         @Override public int heapSizePercentage() { return heapPercentage; }
@@ -307,6 +309,7 @@ public class ModelContextImpl implements ModelContext {
         @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; }
         @Override public boolean enableGlobalPhase() { return enableGlobalPhase; }
         @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown.test(id); }
+        @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; }
 
         private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) {
             return flag.bindTo(source)
diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
index e2810bbede4..23f6c604efd 100644
--- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
+++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java
@@ -414,6 +414,15 @@ public class Flags {
             "Takes effect at next host-admin tick",
             ZONE_ID);
 
+    public static final UnboundBooleanFlag ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR = defineFeatureFlag(
+            "enable-conditional-put-remove-write-repair", false,
+            List.of("vekterli", "havardpe"), "2023-05-10", "2023-07-01",
+            "If set, a conditional Put or Remove operation for a document in an inconsistent bucket " +
+            "will initiate a write-repair that evaluates the condition across all mutually inconsistent " +
+            "replicas, with the newest document version (if any) being authoritative",
+            "Takes effect at redeployment",
+            ZONE_ID, APPLICATION_ID);
+
     /** WARNING: public for testing: All flags should be defined in {@link Flags}. */
     public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners,
                                                        String createdAt, String expiresAt, String description,