diff options
author | Tor Brede Vekterli <vekterli@yahooinc.com> | 2022-08-24 15:00:43 +0200 |
---|---|---|
committer | Tor Brede Vekterli <vekterli@yahooinc.com> | 2022-08-24 15:00:43 +0200 |
commit | 2380b98596ffcf07150a2935858c4204849e199c (patch) | |
tree | 6956e40414e2562a76fd0580f9fb9835e2e26673 | |
parent | 26d0b997cc573bac2a1d7eda7a2494449452e121 (diff) |
Add feature flag for two-phase document GC in content clusters
6 files changed, 51 insertions, 2 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index e45ab5de253..f439e88f260 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -129,6 +129,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"hmusum"}) default Architecture adminClusterArchitecture() { return Architecture.getDefault(); } @ModelFeatureFlag(owners = {"tokle"}) default boolean enableProxyProtocolMixedMode() { return true; } @ModelFeatureFlag(owners = {"arnej"}) default String logFileCompressionAlgorithm(String defVal) { return defVal; } + @ModelFeatureFlag(owners = {"vekterli"}) default boolean useTwoPhaseDocumentGc() { return false; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 8e2f3feb010..c2218cef684 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -79,6 +79,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean enableBitVectors = false; private boolean loadCodeAsHugePages = false; private boolean sharedStringRepoNoReclaim = false; + private boolean useTwoPhaseDocumentGc = false; private int mbus_java_num_targets = 1; private int mbus_java_events_before_wakeup = 1; private int mbus_cpp_num_targets = 1; @@ -149,6 +150,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public int rpcNumTargets() { return rpc_num_targets; } @Override public int rpcEventsBeforeWakeup() { return rpc_events_before_wakeup; } @Override public String queryDispatchPolicy() { return queryDispatchPolicy; } + @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } public TestProperties sharedStringRepoNoReclaim(boolean sharedStringRepoNoReclaim) { @@ -413,6 +415,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties setUseTwoPhaseDocumentGc(boolean useTwoPhase) { + this.useTwoPhaseDocumentGc = useTwoPhase; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java index b8d2a4f91fe..b8e27d92f8c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/DistributorCluster.java @@ -43,6 +43,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl private final boolean useThreePhaseUpdates; private final int maxActivationInhibitedOutOfSyncGroups; private final boolean unorderedMergeChaining; + private final boolean useTwoPhaseDocumentGc; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<DistributorCluster> { @@ -106,11 +107,12 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl boolean useThreePhaseUpdates = deployState.getProperties().featureFlags().useThreePhaseUpdates(); int maxInhibitedGroups = deployState.getProperties().featureFlags().maxActivationInhibitedOutOfSyncGroups(); boolean unorderedMergeChaining = deployState.getProperties().featureFlags().unorderedMergeChaining(); + boolean useTwoPhaseDocumentGc = deployState.getProperties().featureFlags().useTwoPhaseDocumentGc(); return new DistributorCluster(parent, new BucketSplitting.Builder().build(new ModelElement(producerSpec)), gc, hasIndexedDocumentType, useThreePhaseUpdates, - maxInhibitedGroups, unorderedMergeChaining); + maxInhibitedGroups, unorderedMergeChaining, useTwoPhaseDocumentGc); } } @@ -118,7 +120,8 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl GcOptions gc, boolean hasIndexedDocumentType, boolean useThreePhaseUpdates, int maxActivationInhibitedOutOfSyncGroups, - boolean unorderedMergeChaining) + boolean unorderedMergeChaining, + boolean useTwoPhaseDocumentGc) { super(parent, "distributor"); this.parent = parent; @@ -128,6 +131,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl this.useThreePhaseUpdates = useThreePhaseUpdates; this.maxActivationInhibitedOutOfSyncGroups = maxActivationInhibitedOutOfSyncGroups; this.unorderedMergeChaining = unorderedMergeChaining; + this.useTwoPhaseDocumentGc = useTwoPhaseDocumentGc; } @Override @@ -142,6 +146,7 @@ public class DistributorCluster extends AbstractConfigProducer<Distributor> impl builder.enable_metadata_only_fetch_phase_for_inconsistent_updates(useThreePhaseUpdates); builder.max_activation_inhibited_out_of_sync_groups(maxActivationInhibitedOutOfSyncGroups); builder.use_unordered_merge_chaining(unorderedMergeChaining); + builder.enable_two_phase_garbage_collection(useTwoPhaseDocumentGc); bucketSplitting.getConfig(builder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 17ce43bba2c..ac291fc578f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -1165,6 +1165,32 @@ public class ContentClusterTest extends ContentBaseTest { return (new StorDistributormanagerConfig(builder)).use_unordered_merge_chaining(); } + private boolean resolveTwoPhaseGcConfigWithFeatureFlag(Boolean flagEnableTwoPhase) { + var props = new TestProperties(); + if (flagEnableTwoPhase != null) { + props.setUseTwoPhaseDocumentGc(flagEnableTwoPhase); + } + VespaModel model = createEnd2EndOneNode(props); + + ContentCluster cc = model.getContentClusters().get("storage"); + var builder = new StorDistributormanagerConfig.Builder(); + cc.getDistributorNodes().getConfig(builder); + + return (new StorDistributormanagerConfig(builder)).enable_two_phase_garbage_collection(); + } + + @Test + public void two_phase_garbage_collection_config_is_controlled_by_properties() { + assertFalse(resolveTwoPhaseGcConfigWithFeatureFlag(false)); + assertTrue(resolveTwoPhaseGcConfigWithFeatureFlag(true)); + } + + // TODO change once gradual rollout complete + @Test + public void two_phase_garbage_collection_config_is_disabled_by_default() { + assertFalse(resolveTwoPhaseGcConfigWithFeatureFlag(null)); + } + @Test void testDedicatedClusterControllers() { VespaModel noContentModel = createEnd2EndOneNode(new TestProperties().setHostedVespa(true) diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 7c7a12bbf36..69551c62840 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -211,6 +211,7 @@ public class ModelContextImpl implements ModelContext { private final boolean enableProxyProtocolMixedMode; private final boolean sharedStringRepoNoReclaim; private final String logFileCompressionAlgorithm; + private final boolean useTwoPhaseDocumentGc; private final boolean mbus_dispatch_on_decode; private final boolean mbus_dispatch_on_encode; private final int mbus_threads; @@ -271,6 +272,7 @@ public class ModelContextImpl implements ModelContext { this.enableProxyProtocolMixedMode = flagValue(source, appId, version, Flags.ENABLE_PROXY_PROTOCOL_MIXED_MODE); this.sharedStringRepoNoReclaim = flagValue(source, appId, version, Flags.SHARED_STRING_REPO_NO_RECLAIM); this.logFileCompressionAlgorithm = flagValue(source, appId, version, Flags.LOG_FILE_COMPRESSION_ALGORITHM); + this.useTwoPhaseDocumentGc = flagValue(source, appId, version, Flags.USE_TWO_PHASE_DOCUMENT_GC); this.mbus_java_num_targets = flagValue(source, appId, version, Flags.MBUS_JAVA_NUM_TARGETS); this.mbus_java_events_before_wakeup = flagValue(source, appId, version, Flags.MBUS_JAVA_EVENTS_BEFORE_WAKEUP); this.mbus_cpp_num_targets = flagValue(source, appId, version, Flags.MBUS_CPP_NUM_TARGETS); @@ -343,6 +345,7 @@ public class ModelContextImpl implements ModelContext { } return defVal; } + @Override public boolean useTwoPhaseDocumentGc() { return useTwoPhaseDocumentGc; } private static <V> V flagValue(FlagSource source, ApplicationId appId, Version vespaVersion, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 21f95c7ac6a..5527a565cf4 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -488,6 +488,13 @@ public class Flags { "Takes effect next maintenance run" ); + public static final UnboundBooleanFlag USE_TWO_PHASE_DOCUMENT_GC = defineFeatureFlag( + "use-two-phase-document-gc", false, + List.of("vekterli"), "2022-08-24", "2022-11-01", + "Use two-phase document GC in content clusters", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, |