diff options
author | Tor Brede Vekterli <vekterli@yahooinc.com> | 2023-05-10 15:44:45 +0200 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@yahooinc.com> | 2023-05-10 15:44:45 +0200 |
commit | 2ad5edd01b4eea8aa961413d10d3777012adc53e (patch) | |
tree | 834173b4604ee3ef565af79d42e80e3e16418827 | |
parent | 2d11f545108d46dc6ae2aff098892f5462f2cefc (diff) |
Add feature flag for enabling conditional put/remove write repair
This maps to controlling the distributed condition probe feature
of the content layer, which in turn controls whether a conditional
put/remove to an inconsistent bucket triggers a write repair step
on the distributor.
6 files changed, 39 insertions, 2 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index ee7ca64166d..41ee47513a0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -115,6 +115,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"arnej, bjorncs"}) default boolean enableGlobalPhase() { return true; } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select summary decode type") default String summaryDecodePolicy() { return "eager"; } @ModelFeatureFlag(owners = {"hmusum"}) default boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return false; } + @ModelFeatureFlag(owners = {"vekterli", "havardpe"}) default boolean enableConditionalPutRemoveWriteRepair() { return false; } //Below are all flags that must be kept until 7 is out of the door @ModelFeatureFlag(owners = {"arnej"}, removeAfter="7.last") default boolean ignoreThreadStackSizes() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 540905bef4c..56f999f85b4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -84,6 +84,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private Optional<CloudAccount> cloudAccount = Optional.empty(); private boolean allowUserFilters = true; private boolean allowMoreThanOneContentGroupDown = false; + private boolean enableConditionalPutRemoveWriteRepair = false; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -142,6 +143,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean allowUserFilters() { return allowUserFilters; } @Override public boolean enableGlobalPhase() { return true; } // Enable global-phase by default for unit tests only @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown; } + @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { this.sharedStringRepoNoReclaim = sharedStringRepoNoReclaim; @@ -375,6 +377,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setEnableConditionalPutRemoveWriteRepair(boolean enable) { + this.enableConditionalPutRemoveWriteRepair = enable; + return this; + } + public TestProperties setAllowUserFilters(boolean b) { this.allowUserFilters = b; return this; } public static class Spec implements ConfigServerSpec { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index d73f1618dbc..4aac8bfb647 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -33,6 +33,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen private final GcOptions gc; private final boolean hasIndexedDocumentType; private final int maxActivationInhibitedOutOfSyncGroups; + private final boolean enableConditionalPutRemoveWriteRepair; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<DistributorCluster> { ContentCluster parent; @@ -93,17 +94,20 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen final GcOptions gc = parseGcOptions(documentsNode); final boolean hasIndexedDocumentType = clusterContainsIndexedDocumentType(documentsNode); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); + boolean enableConditionalPutRemoveWriteRepair = deployState.getProperties().featureFlags().enableConditionalPutRemoveWriteRepair(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, - maxInhibitedGroups); + maxInhibitedGroups, + enableConditionalPutRemoveWriteRepair); } } private DistributorCluster(ContentCluster parent, BucketSplitting bucketSplitting, GcOptions gc, boolean hasIndexedDocumentType, - int maxActivationInhibitedOutOfSyncGroups) + int maxActivationInhibitedOutOfSyncGroups, + boolean enableConditionalPutRemoveWriteRepair) { super(parent, "distributor"); this.parent = parent; @@ -111,6 +115,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen this.gc = gc; this.hasIndexedDocumentType = hasIndexedDocumentType; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; + this.enableConditionalPutRemoveWriteRepair = enableConditionalPutRemoveWriteRepair; } @Override @@ -123,6 +128,7 @@ public class DistributorCluster extends TreeConfigProducer<Distributor> implemen builder.enable_revert(parent.getPersistence().supportRevert()); builder.disable_bucket_activation(!hasIndexedDocumentType); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); + builder.enable_condition_probing(enableConditionalPutRemoveWriteRepair); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 71dd3223da2..14fe7bbcc36 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1240,6 +1240,17 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(2, resolveMaxInhibitedGroupsConfigWithFeatureFlag(2)); } + private boolean resolveConditionProbingFromWriteRepairFeatureFlag(boolean enable) throws Exception { + var cfg = resolveStorDistributormanagerConfig(new TestProperties().setEnableConditionalPutRemoveWriteRepair(enable)); + return cfg.enable_condition_probing(); + } + + @Test + void distributor_condition_probing_is_controlled_by_write_repair_feature_flag() throws Exception { + assertFalse(resolveConditionProbingFromWriteRepairFeatureFlag(false)); + assertTrue(resolveConditionProbingFromWriteRepairFeatureFlag(true)); + } + private int resolveNumDistributorStripesConfig(Optional<Flavor> flavor) throws Exception { var cc = createOneNodeCluster(new TestProperties(), flavor); var builder = new StorDistributormanagerConfig.Builder(); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index b443a0e64b0..59ddb8b68ab 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -205,6 +205,7 @@ public class ModelContextImpl implements ModelContext { private final boolean enableGlobalPhase; private final String summaryDecodePolicy; private final Predicate<ClusterSpec.Id> allowMoreThanOneContentGroupDown; + private final boolean enableConditionalPutRemoveWriteRepair; public FeatureFlags(FlagSource source, ApplicationId appId, Version version) { this.defaultTermwiseLimit = flagValue(source, appId, version, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -252,6 +253,7 @@ public class ModelContextImpl implements ModelContext { this.enableGlobalPhase = flagValue(source, appId, version, Flags.ENABLE_GLOBAL_PHASE); this.summaryDecodePolicy = flagValue(source, appId, version, Flags.SUMMARY_DECODE_POLICY); this.allowMoreThanOneContentGroupDown = clusterId -> flagValue(source, appId, version, clusterId, Flags.ALLOW_MORE_THAN_ONE_CONTENT_GROUP_DOWN); + this.enableConditionalPutRemoveWriteRepair = flagValue(source, appId, version, Flags.ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR); } @Override public int heapSizePercentage() { return heapPercentage; } @@ -307,6 +309,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useRestrictedDataPlaneBindings() { return useRestrictedDataPlaneBindings; } @Override public boolean enableGlobalPhase() { return enableGlobalPhase; } @Override public boolean allowMoreThanOneContentGroupDown(ClusterSpec.Id id) { return allowMoreThanOneContentGroupDown.test(id); } + @Override public boolean enableConditionalPutRemoveWriteRepair() { return enableConditionalPutRemoveWriteRepair; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index e2810bbede4..23f6c604efd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -414,6 +414,15 @@ public class Flags { "Takes effect at next host-admin tick", ZONE_ID); + public static final UnboundBooleanFlag ENABLE_CONDITIONAL_PUT_REMOVE_WRITE_REPAIR = defineFeatureFlag( + "enable-conditional-put-remove-write-repair", false, + List.of("vekterli", "havardpe"), "2023-05-10", "2023-07-01", + "If set, a conditional Put or Remove operation for a document in an inconsistent bucket " + + "will initiate a write-repair that evaluates the condition across all mutually inconsistent " + + "replicas, with the newest document version (if any) being authoritative", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, |