diff options
7 files changed, 43 insertions, 0 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 3df93f7d08d..1f78ad20e40 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -110,6 +110,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"vekterli", "geirst"}) default boolean unorderedMergeChaining() { return false; } @ModelFeatureFlag(owners = {"arnej"}) default boolean useV8GeoPositions() { return false; } @ModelFeatureFlag(owners = {"arnej", "baldersheim"}) default boolean useV8DocManagerCfg() { return false; } + @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default int maxCompactBuffers() { return 1; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index ee9af73c554..cf649162c08 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -75,6 +75,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean asyncApplyBucketDiff = false; private boolean unorderedMergeChaining = false; private List<String> zoneDnsSuffixes = List.of(); + private int maxCompactBuffers = 1; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -130,6 +131,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @Override public boolean unorderedMergeChaining() { return unorderedMergeChaining; } @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } + @Override public int maxCompactBuffers() { return maxCompactBuffers; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; @@ -340,6 +342,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties maxCompactBuffers(int maxCompactBuffers) { + this.maxCompactBuffers = maxCompactBuffers; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 13c3c229acb..f981ed6228f 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -71,6 +71,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final double defaultDiskBloatFactor; private final int defaultDocStoreCompressionLevel; private final boolean forwardIssuesToQrs; + private final int defaultMaxCompactBuffers; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -225,6 +226,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); + this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } public void setVisibilityDelay(double delay) { @@ -387,6 +389,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); + ddbB.allocation.max_compact_buffers(defaultMaxCompactBuffers); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 729348a0e3a..357ae7e2656 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -47,6 +47,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.OptionalInt; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; @@ -1026,6 +1027,27 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } + private int resolveMaxCompactBuffers(OptionalInt maxCompactBuffers) { + TestProperties testProperties = new TestProperties(); + if (maxCompactBuffers.isPresent()) { + testProperties.maxCompactBuffers(maxCompactBuffers.getAsInt()); + } + VespaModel model = createEnd2EndOneNode(testProperties); + ContentCluster cc = model.getContentClusters().get("storage"); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + assertEquals(1, protonConfig.documentdb().size()); + return protonConfig.documentdb(0).allocation().max_compact_buffers(); + } + + @Test + public void default_max_compact_buffers_config_controlled_by_properties() { + assertEquals(1, resolveMaxCompactBuffers(OptionalInt.empty())); + assertEquals(2, resolveMaxCompactBuffers(OptionalInt.of(2))); + assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); + } + void assertZookeeperServerImplementation(String expectedClassName, ClusterControllerContainerCluster clusterControllerCluster) { for (ClusterControllerContainer c : clusterControllerCluster.getContainers()) { diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 1b55d17fd36..978241339d2 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -202,6 +202,7 @@ public class ModelContextImpl implements ModelContext { private final boolean unorderedMergeChaining; private final boolean useV8GeoPositions; private final boolean useV8DocManagerCfg; + private final int maxCompactBuffers; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -242,6 +243,7 @@ public class ModelContextImpl implements ModelContext { this.unorderedMergeChaining = flagValue(source, appId, Flags.UNORDERED_MERGE_CHAINING); this.useV8GeoPositions = flagValue(source, appId, Flags.USE_V8_GEO_POSITIONS); this.useV8DocManagerCfg = flagValue(source, appId, Flags.USE_V8_DOC_MANAGER_CFG); + this.maxCompactBuffers = flagValue(source, appId, Flags.MAX_COMPACT_BUFFERS); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } diff --git a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java index b0046a201ab..0e8c69327e2 100644 --- a/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java +++ b/configserver/src/test/java/com/yahoo/vespa/config/server/ModelContextImplTest.java @@ -102,6 +102,7 @@ public class ModelContextImplTest { assertEquals(1.0, context.properties().featureFlags().defaultTermwiseLimit(), 0.0); assertFalse(context.properties().featureFlags().useAsyncMessageHandlingOnSchedule()); assertEquals(0.5, context.properties().featureFlags().feedConcurrency(), 0.0); + assertEquals(1, context.properties().featureFlags().maxCompactBuffers()); } } diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index eb68c72eb6a..97b06c802dd 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -379,6 +379,13 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag MAX_COMPACT_BUFFERS = defineIntFlag( + "max-compact-buffers", 1, + List.of("baldersheim", "geirst", "toregge"), "2021-12-15", "2022-03-31", + "Upper limit of buffers to compact in a data store at the same time for each reason (memory usage, address space usage)", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, String createdAt, String expiresAt, String description, |