diff options
author | Geir Storli <geirst@verizonmedia.com> | 2021-11-05 13:47:18 +0000 |
---|---|---|
committer | Geir Storli <geirst@verizonmedia.com> | 2021-11-05 13:47:18 +0000 |
commit | b994f19b89973f75664ce71f1dad58ffd541f246 (patch) | |
tree | 18400b332631a195557ef74a27f987a144c8a458 | |
parent | c7d50a638da8d991254c3cc3c151d95ea5ef8b51 (diff) |
Add feature flag for whether to use a shared field writer executor for the document database(s) in proton.
6 files changed, 55 insertions, 0 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 8c2957502a8..ff612ffc2b0 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -73,6 +73,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli"}) default boolean useThreePhaseUpdates() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}, comment = "Select sequencer type use while feeding") default String feedSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"geirst, baldersheim"}) default int feedTaskLimit() { return 1000; } + @ModelFeatureFlag(owners = {"geirst, baldersheim"}) default String sharedFieldWriterExecutor() { return "NONE"; } @ModelFeatureFlag(owners = {"baldersheim"}) default String responseSequencerType() { throw new UnsupportedOperationException("TODO specify default value"); } @ModelFeatureFlag(owners = {"baldersheim"}) default int defaultNumResponseThreads() { return 2; } @ModelFeatureFlag(owners = {"baldersheim"}) default boolean skipCommunicationManagerThread() { throw new UnsupportedOperationException("TODO specify default value"); } diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index b7506587102..5183a3ca587 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -42,6 +42,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private String jvmGCOptions = null; private String sequencerType = "LATENCY"; private int feedTaskLimit = 1000; + private String sharedFieldWriterExecutor = "NONE"; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; @@ -85,6 +86,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { return jvmGCOptions; } @Override public String feedSequencerType() { return sequencerType; } @Override public int feedTaskLimit() { return feedTaskLimit; } + @Override public String sharedFieldWriterExecutor() { return sharedFieldWriterExecutor; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return firstTimeDeployment; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @@ -172,6 +174,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea feedTaskLimit = value; return this; } + public TestProperties setSharedFieldWriterExecutor(String value) { + sharedFieldWriterExecutor = value; + return this; + } public TestProperties setResponseSequencerType(String type) { responseSequencerType = type; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index ff9db4a10bf..917b1d42146 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,6 +65,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final int feedTaskLimit; + private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; private final double defaultDiskBloatFactor; private final int defaultDocStoreCompressionLevel; @@ -194,6 +195,14 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } } + private static ProtonConfig.Feeding.Shared_field_writer_executor.Enum convertSharedFieldWriterExecutor(String value) { + try { + return ProtonConfig.Feeding.Shared_field_writer_executor.Enum.valueOf(value); + } catch (Throwable t) { + return ProtonConfig.Feeding.Shared_field_writer_executor.Enum.NONE; + } + } + private ContentSearchCluster(AbstractConfigProducer<?> parent, String clusterName, ModelContext.FeatureFlags featureFlags, @@ -213,6 +222,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); this.feedTaskLimit = featureFlags.feedTaskLimit(); + this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); @@ -435,6 +445,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> builder.indexing.optimize(feedSequencerType); } builder.indexing.tasklimit(feedTaskLimit); + builder.feeding.shared_field_writer_executor(sharedFieldWriterExecutor); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 5395b0868cb..1de37dbeccf 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -34,6 +34,7 @@ import java.util.List; import static com.yahoo.config.model.api.container.ContainerServiceType.CLUSTERCONTROLLER_CONTAINER; import static com.yahoo.config.model.api.container.ContainerServiceType.METRICS_PROXY_CONTAINER; +import static com.yahoo.vespa.config.search.core.ProtonConfig.Feeding.Shared_field_writer_executor; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.Matchers.containsString; import static org.junit.Assert.assertEquals; @@ -823,6 +824,31 @@ public class ContentBuilderTest extends DomBuilderTest { return resolveProtonConfig(props, singleNodeContentXml()).indexing().tasklimit(); } + @Test + public void shared_field_writer_executor_is_controlled_by_feature_flag() { + + assertEquals(Shared_field_writer_executor.Enum.NONE, + resolveSharedFieldWriterExecutorConfigWithFeatureFlag(null)); + assertEquals(Shared_field_writer_executor.Enum.NONE, + resolveSharedFieldWriterExecutorConfigWithFeatureFlag("NONE")); + assertEquals(Shared_field_writer_executor.Enum.INDEX, + resolveSharedFieldWriterExecutorConfigWithFeatureFlag("INDEX")); + assertEquals(Shared_field_writer_executor.Enum.INDEX_AND_ATTRIBUTE, + resolveSharedFieldWriterExecutorConfigWithFeatureFlag("INDEX_AND_ATTRIBUTE")); + assertEquals(Shared_field_writer_executor.Enum.DOCUMENT_DB, + resolveSharedFieldWriterExecutorConfigWithFeatureFlag("DOCUMENT_DB")); + assertEquals(Shared_field_writer_executor.Enum.NONE, + resolveSharedFieldWriterExecutorConfigWithFeatureFlag("invalid")); + } + + private ProtonConfig.Feeding.Shared_field_writer_executor.Enum resolveSharedFieldWriterExecutorConfigWithFeatureFlag(String value) { + var props = new TestProperties(); + if (value != null) { + props.setSharedFieldWriterExecutor(value); + } + return resolveProtonConfig(props, singleNodeContentXml()).feeding().shared_field_writer_executor(); + } + private void verifyThatFeatureFlagControlsVisibilityDelayDefault(Double xmlOverride, double expected) { String hostedXml = xmlWithVisibilityDelay(xmlOverride); var config = resolveProtonConfig(new TestProperties(), hostedXml); diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 98e65724468..a824433ae69 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -168,6 +168,7 @@ public class ModelContextImpl implements ModelContext { private final boolean useThreePhaseUpdates; private final String feedSequencer; private final int feedTaskLimit; + private final String sharedFieldWriterExecutor; private final String responseSequencer; private final int numResponseThreads; private final boolean skipCommunicationManagerThread; @@ -203,6 +204,7 @@ public class ModelContextImpl implements ModelContext { this.useThreePhaseUpdates = flagValue(source, appId, Flags.USE_THREE_PHASE_UPDATES); this.feedSequencer = flagValue(source, appId, Flags.FEED_SEQUENCER_TYPE); this.feedTaskLimit = flagValue(source, appId, Flags.FEED_TASK_LIMIT); + this.sharedFieldWriterExecutor = flagValue(source, appId, Flags.SHARED_FIELD_WRITER_EXECUTOR); this.responseSequencer = flagValue(source, appId, Flags.RESPONSE_SEQUENCER_TYPE); this.numResponseThreads = flagValue(source, appId, Flags.RESPONSE_NUM_THREADS); this.skipCommunicationManagerThread = flagValue(source, appId, Flags.SKIP_COMMUNICATIONMANAGER_THREAD); @@ -238,6 +240,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean useThreePhaseUpdates() { return useThreePhaseUpdates; } @Override public String feedSequencerType() { return feedSequencer; } @Override public int feedTaskLimit() { return feedTaskLimit; } + @Override public String sharedFieldWriterExecutor() { return sharedFieldWriterExecutor; } @Override public String responseSequencerType() { return responseSequencer; } @Override public int defaultNumResponseThreads() { return numResponseThreads; } @Override public boolean skipCommunicationManagerThread() { return skipCommunicationManagerThread; } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 060a862988c..d72582287a9 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -86,6 +86,14 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundStringFlag SHARED_FIELD_WRITER_EXECUTOR = defineStringFlag( + "shared-field-writer-executor", "NONE", + List.of("geirst, baldersheim"), "2021-11-05", "2022-02-01", + "Whether to use a shared field writer executor for the document database(s) in proton. " + + "Valid values: NONE, INDEX, INDEX_AND_ATTRIBUTE, DOCUMENT_DB", + "Takes effect at redeployment (requires restart)", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MAX_UNCOMMITTED_MEMORY = defineIntFlag( "max-uncommitted-memory", 130000, List.of("geirst, baldersheim"), "2021-10-21", "2022-01-01", |