diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2021-10-14 18:08:29 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-10-14 18:08:29 +0200 |
commit | a03ae4fb37cfddd3183f85e878db3dad13dabf71 (patch) | |
tree | 84192c8799841c2bddc51f4e1a4a1c478c21d5f3 /config-model | |
parent | ffcc28f2b9180ad8774c67fddbebc81b41f2fc6c (diff) | |
parent | 3a6b4cae8ea09a6de602c67b045bf3ab6cb755b0 (diff) |
Merge pull request #19572 from vespa-engine/geirst/feed-task-limit-feature-flag
Add feature flag for the task limit used by the executors handling fe…
Diffstat (limited to 'config-model')
3 files changed, 37 insertions, 16 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index a728dfe71b9..68f7b0cc4ec 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -41,6 +41,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private double defaultTermwiseLimit = 1.0; private String jvmGCOptions = null; private String sequencerType = "LATENCY"; + private int feedTaskLimit = 1000; private boolean firstTimeDeployment = false; private String responseSequencerType = "ADAPTIVE"; private int responseNumThreads = 2; @@ -81,6 +82,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public Set<ContainerEndpoint> endpoints() { return endpoints; } @Override public String jvmGCOptions(Optional<ClusterSpec.Type> clusterType) { return jvmGCOptions; } @Override public String feedSequencerType() { return sequencerType; } + @Override public int feedTaskLimit() { return feedTaskLimit; } @Override public boolean isBootstrap() { return false; } @Override public boolean isFirstTimeDeployment() { return firstTimeDeployment; } @Override public boolean useDedicatedNodeForLogserver() { return useDedicatedNodeForLogserver; } @@ -157,6 +159,10 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea sequencerType = type; return this; } + public TestProperties setFeedTaskLimit(int value) { + feedTaskLimit = value; + return this; + } public TestProperties setResponseSequencerType(String type) { responseSequencerType = type; return this; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 3ab7e947a2f..1200b7467b4 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -64,6 +64,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; + private final int feedTaskLimit; private final double defaultFeedConcurrency; private final double defaultDiskBloatFactor; private final int defaultDocStoreCompressionLevel; @@ -210,6 +211,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); + feedTaskLimit = featureFlags.feedTaskLimit(); defaultFeedConcurrency = featureFlags.feedConcurrency(); defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); defaultDiskBloatFactor = featureFlags.diskBloatFactor(); @@ -429,6 +431,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.indexing.optimize(feedSequencerType); } + builder.indexing.tasklimit(feedTaskLimit); } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java index 891a870dc01..5395b0868cb 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/ContentBuilderTest.java @@ -777,21 +777,24 @@ public class ContentBuilderTest extends DomBuilderTest { "</services>"; } + private ProtonConfig resolveProtonConfig(TestProperties props, String hostedXml) { + var deployStateBuilder = new DeployState.Builder().properties(props); + var model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() + .withServices(hostedXml) + .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) + .build()) + .create(deployStateBuilder); + return getProtonConfig(model.getContentClusters().values().iterator().next()); + } + private void verifyFeedSequencer(String input, String expected) { verifyFeedSequencer(input, expected, 0); } + private void verifyFeedSequencer(String input, String expected, double visibilityDelay) { String hostedXml = xmlWithVisibilityDelay(visibilityDelay); - - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties().setFeedSequencerType(input)); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(hostedXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + var config = resolveProtonConfig(new TestProperties().setFeedSequencerType(input), hostedXml); assertEquals(expected, config.indexing().optimize().toString()); - } @Test @@ -806,17 +809,26 @@ public class ContentBuilderTest extends DomBuilderTest { } + @Test + public void feed_task_limit_is_controlled_by_feature_flag() { + assertEquals(1000, resolveFeedTaskLimitConfigWithFeatureFlag(null)); + assertEquals(2000, resolveFeedTaskLimitConfigWithFeatureFlag(2000)); + } + + private int resolveFeedTaskLimitConfigWithFeatureFlag(Integer value) { + var props = new TestProperties(); + if (value != null) { + props.setFeedTaskLimit(value); + } + return resolveProtonConfig(props, singleNodeContentXml()).indexing().tasklimit(); + } + private void verifyThatFeatureFlagControlsVisibilityDelayDefault(Double xmlOverride, double expected) { String hostedXml = xmlWithVisibilityDelay(xmlOverride); - DeployState.Builder deployStateBuilder = new DeployState.Builder().properties(new TestProperties()); - VespaModel model = new VespaModelCreatorWithMockPkg(new MockApplicationPackage.Builder() - .withServices(hostedXml) - .withSearchDefinition(MockApplicationPackage.MUSIC_SEARCHDEFINITION) - .build()) - .create(deployStateBuilder); - ProtonConfig config = getProtonConfig(model.getContentClusters().values().iterator().next()); + var config = resolveProtonConfig(new TestProperties(), hostedXml); assertEquals(expected, config.documentdb(0).visibilitydelay(), 0.0); } + @Test public void verifyThatFeatureFlagControlsVisibilityDelayDefault() { verifyThatFeatureFlagControlsVisibilityDelayDefault(null, 0.0); |