diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-01-24 00:02:34 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-01-24 00:02:34 +0100 |
commit | b04cbff7a7d15dffeaa4230c2e6a1c55af66b4d0 (patch) | |
tree | d497076856672b306436f51c8bffc8ef21635dec | |
parent | f25dbc75d2b98d01e61c4d4387005629483fdc1e (diff) | |
parent | c2f9154beb850b7cbadcd3f974e4869db7953f1d (diff) |
Merge pull request #16188 from vespa-engine/balder/control-lidspace-job
Control which lidspace job implementation to use.
6 files changed, 50 insertions, 10 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index d020079bb17..e75038a863e 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -80,6 +80,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"baldersheim"}) default int contentNodeBucketDBStripeBits() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int mergeChunkSize() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default double feedConcurrency() { throw new UnsupportedOperationException("TODO specify default value"); } + @ModelFeatureFlag(owners = {"baldersheim"}) default boolean useBucketExecutorForLidSpaceCompact() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"musum", "mpolden"}, comment = "Revisit in February 2021") default boolean reconfigurableZookeeperServer() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default boolean enableJdiscConnectionLog() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "tokle", "baldersheim"}) default boolean enableZstdCompressionAccessLog() { return false; } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 9e8e9656e90..92d425a9d4e 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -26,7 +26,6 @@ import java.util.Set; * * @author hakonhall */ -@SuppressWarnings("deprecation") public class TestProperties implements ModelContext.Properties, ModelContext.FeatureFlags { private boolean multitenant = false; @@ -53,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double feedConcurrency = 0.5; private boolean enableAutomaticReindexing = false; private boolean reconfigurableZookeeperServer = false; + private boolean useBucketExecutorForLidSpaceCompact; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -87,6 +87,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double feedConcurrency() { return feedConcurrency; } @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } + @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; @@ -195,6 +196,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties useBucketExecutorForLidSpaceCompact(boolean enabled) { + useBucketExecutorForLidSpaceCompact = enabled; + return this; + } + public static class Spec implements ConfigServerSpec { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 79b98837238..dd29df61f35 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,6 +65,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; + private final boolean useBucketExecutorForLidSpaceCompact; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -205,6 +206,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); + useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact(); } public void setVisibilityDelay(double delay) { @@ -425,6 +427,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } + builder.lidspacecompaction.usebucketexecutor(useBucketExecutorForLidSpaceCompact); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 00b3fc22956..55234a3a96e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -719,6 +719,17 @@ public class ContentBuilderTest extends DomBuilderTest { assertThat(config.search().mmap().options(0), is(ProtonConfig.Search.Mmap.Options.POPULATE)); } + private String singleNodeContentXml() { + return "<services>" + + "<content version='1.0' id='search'>" + + " <redundancy>1</redundancy>" + + " <documents>" + + " <document type='music' mode='index'/>" + + " </documents>" + + " <nodes count='1'/>" + + "</content>" + + "</services>"; + } @Test public void ensurePruneRemovedDocumentsAgeForHostedVespa() { { @@ -738,15 +749,7 @@ public class ContentBuilderTest extends DomBuilderTest { } { - String hostedXml = "<services>" + - "<content version='1.0' id='search'>" + - " <redundancy>1</redundancy>" + - " <documents>" + - " <document type='music' mode='index'/>" + - " </documents>" + - " <nodes count='1'/>" + - "</content>" + - "</services>"; + String hostedXml = singleNodeContentXml(); DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setHostedVespa(true)); VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() @@ -821,6 +824,23 @@ public class ContentBuilderTest extends DomBuilderTest { verifyThatFeatureFlagControlsVisibilityDelayDefault(0.6, 0.6); } + private void verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(boolean flag) { + DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().useBucketExecutorForLidSpaceCompact(flag)); + VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(singleNodeContentXml()) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + assertEquals(flag, config.lidspacecompaction().usebucketexecutor()); + } + + @Test + public void verifyUseBucketExecutorForLidSpaceCompact() { + verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(true); + verifyThatFeatureFlagControlsUseBucketExecutorForLidSpaceCompact(false); + } + @Test public void failWhenNoDocumentsElementSpecified() { expectedException.expect(IllegalArgumentException.class); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 596f9c0ce98..e1642efcd44 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -165,6 +165,7 @@ public class ModelContextImpl implements ModelContext { private final boolean reconfigurableZookeeperServer; private final boolean enableJdiscConnectionLog; private final boolean enableZstdCompressionAccessLog; + private final boolean useBucketExecutorForLidSpaceCompact; public FeatureFlags(FlagSource source, ApplicationId appId) { this.enableAutomaticReindexing = flagValue(source, appId, Flags.ENABLE_AUTOMATIC_REINDEXING); @@ -185,6 +186,7 @@ public class ModelContextImpl implements ModelContext { this.reconfigurableZookeeperServer = flagValue(source, appId, Flags.RECONFIGURABLE_ZOOKEEPER_SERVER_FOR_CLUSTER_CONTROLLER); this.enableJdiscConnectionLog = flagValue(source, appId, Flags.ENABLE_JDISC_CONNECTION_LOG); this.enableZstdCompressionAccessLog = flagValue(source, appId, Flags.ENABLE_ZSTD_COMPRESSION_ACCESS_LOG); + this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT); } @Override public boolean enableAutomaticReindexing() { return enableAutomaticReindexing; } @@ -205,6 +207,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean enableJdiscConnectionLog() { return enableJdiscConnectionLog; } @Override public boolean enableZstdCompressionAccessLog() { return enableZstdCompressionAccessLog; } + @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 6f7970e1357..7c2deeda69b 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -271,6 +271,13 @@ public class Flags { "Takes effect on (re)deployment", APPLICATION_ID); + public static final UnboundBooleanFlag USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT = defineFeatureFlag( + "use-bucket-executor-for-lid-space-compact", false, + List.of("baldersheim"), "2021-01-24", "2021-03-01", + "Wheter to use content-level bucket executor or legacy frozen buckets", + "Takes effect on next internal redeployment", + APPLICATION_ID); + public static final UnboundBooleanFlag USE_POWER_OF_TWO_CHOICES_LOAD_BALANCING = defineFeatureFlag( "use-power-of-two-choices-load-balancing", false, List.of("tokle"), "2020-12-02", "2021-02-01", |