diff options
6 files changed, 38 insertions, 0 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3a2f71e2b8c..fccb8c2891a 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -82,6 +82,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForBucketMove() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForPruneRemoved() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return true; } @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.2; } @ModelFeatureFlag(owners = {"hmusum"}) default int clusterControllerMaxHeapSizeInMb() { return 128; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 75a1a167446..810b2db584a 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -53,6 +53,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedConcurrency = 0.5; private boolean useBucketExecutorForLidSpaceCompact; private boolean useBucketExecutorForBucketMove; + private boolean useBucketExecutorForPruneRemoved; private boolean enableFeedBlockInDistributor = true; private double maxDeadBytesRatio = 0.2; private int clusterControllerMaxHeapSizeInMb = 256; @@ -95,6 +96,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } @Override public boolean useBucketExecutorForBucketMove() { return useBucketExecutorForBucketMove; } + @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } @@ -210,6 +212,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties useBucketExecutorForPruneRemoved(boolean enabled) { + useBucketExecutorForPruneRemoved = enabled; + return this; + } + public TestProperties enableFeedBlockInDistributor(boolean enabled) { enableFeedBlockInDistributor = enabled; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 1dd5074aedb..23cce0c1fb9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -68,6 +68,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final double defaultFeedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; private final boolean useBucketExecutorForBucketMove; + private final boolean useBucketExecutorForPruneRemoved; private final double defaultMaxDeadBytesRatio; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ @@ -216,6 +217,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> defaultFeedConcurrency = featureFlags.feedConcurrency(); useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact(); useBucketExecutorForBucketMove = featureFlags.useBucketExecutorForBucketMove(); + useBucketExecutorForPruneRemoved = featureFlags.useBucketExecutorForPruneRemoved(); defaultMaxDeadBytesRatio = featureFlags.maxDeadBytesRatio(); } @@ -437,6 +439,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> builder.maintenancejobs.maxoutstandingmoveops(maxPendingMoveOps); builder.lidspacecompaction.usebucketexecutor(useBucketExecutorForLidSpaceCompact); builder.bucketmove.usebucketexecutor(useBucketExecutorForBucketMove); + builder.pruneremoveddocuments.usebucketexecutor(useBucketExecutorForPruneRemoved); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 3a3dde0cf87..3168b57100f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -857,6 +857,17 @@ public class ContentBuilderTest extends DomBuilderTest { assertEquals(moveOps, config.maintenancejobs().maxoutstandingmoveops()); } + private void verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(boolean flag) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForPruneRemoved(flag)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(singleNodeContentXml()) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(flag, config.pruneremoveddocuments().usebucketexecutor()); + } + @Test public void verifyMaxPendingMoveOps() { verifyThatFeatureFlagControlsMaxpendingMoveOps(13); @@ -876,6 +887,12 @@ public class ContentBuilderTest extends DomBuilderTest { } @Test + public void verifyUseBucketExecutorForPruneRemoved() { + verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(true); + verifyThatFeatureFlagControlsUseBucketExecutorForPruneRemoved(false); + } + + @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); expectedException.expectMessage("The specified content engine requires the <documents> element to be specified."); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index abd0548e94a..4074fa52ca5 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -171,6 +171,7 @@ public class ModelContextImpl implements ModelContext { private final double feedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; private final boolean useBucketExecutorForBucketMove; + private final boolean useBucketExecutorForPruneRemoved; private final boolean enableFeedBlockInDistributor; private final double maxDeadBytesRatio; private final int clusterControllerMaxHeapSizeInMb; @@ -198,6 +199,7 @@ public class ModelContextImpl implements ModelContext { this.feedConcurrency = flagValue(source, appId, Flags.FEED_CONCURRENCY); this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT); this.useBucketExecutorForBucketMove = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_BUCKET_MOVE); + this.useBucketExecutorForPruneRemoved = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); this.maxDeadBytesRatio = flagValue(source, appId, Flags.MAX_DEAD_BYTES_RATIO); this.clusterControllerMaxHeapSizeInMb = flagValue(source, appId, Flags.CLUSTER_CONTROLLER_MAX_HEAP_SIZE_IN_MB); @@ -225,6 +227,7 @@ public class ModelContextImpl implements ModelContext { @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } @Override public boolean useBucketExecutorForBucketMove() { return useBucketExecutorForBucketMove; } + @Override public boolean useBucketExecutorForPruneRemoved() { return useBucketExecutorForPruneRemoved; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } @Override public int clusterControllerMaxHeapSizeInMb() { return clusterControllerMaxHeapSizeInMb; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d2b7dd90683..fc327b09490 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -165,6 +165,13 @@ public class Flags { "Takes effect on next internal redeployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_PRUNE_REMOVED = defineFeatureFlag( + "use-bucket-executor-for-prune-removed", true, + List.of("baldersheim"), "2021-05-04", "2021-06-01", + "Wheter to use content-level bucket executor or legacy frozen buckets for prune removed", + "Takes effect on next internal redeployment", + APPLICATION_ID); + public static final UnboundBooleanFlag GROUP_SUSPENSION = defineFeatureFlag( "group-suspension", true, List.of("hakon"), "2021-01-22", "2021-05-22", |