diff options
author | Tor Egge <Tor.Egge@broadpark.no> | 2021-02-03 15:54:52 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@broadpark.no> | 2021-02-03 15:54:52 +0100 |
commit | 87e7d5eb3e90cde894eb7183a38a7111e9af048d (patch) | |
tree | 451f33a762f0cc278afb503e1f52c7843f843967 | |
parent | 21d9cda9a3a50ab3186eecbd9e9ac129762640a2 (diff) |
Add feature flag for max dead bytes ratio.
6 files changed, 36 insertions, 0 deletions
diff --git a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java index 6517c3e57fc..bc131b27080 100644 --- a/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java +++ b/config-model-api/src/main/java/com/yahoo/config/model/api/ModelContext.java @@ -84,6 +84,7 @@ public interface ModelContext { @ModelFeatureFlag(owners = {"bjorncs", "tokle"}) default boolean enableJdiscConnectionLog() { return false; } @ModelFeatureFlag(owners = {"bjorncs", "tokle", "baldersheim"}) default boolean enableZstdCompressionAccessLog() { return false; } @ModelFeatureFlag(owners = {"geirst"}) default boolean enableFeedBlockInDistributor() { return false; } + @ModelFeatureFlag(owners = {"baldersheim", "geirst", "toregge"}) default double maxDeadBytesRatio() { return 0.2; } } /** Warning: As elsewhere in this package, do not make backwards incompatible changes that will break old config models! */ diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 6ca1c8f2b79..bcf76c15fa2 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -52,6 +52,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean reconfigurableZookeeperServer = false; private boolean useBucketExecutorForLidSpaceCompact; private boolean enableFeedBlockInDistributor = false; + private double maxDeadBytesRatio = 0.2; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -86,6 +87,7 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public boolean reconfigurableZookeeperServer() { return reconfigurableZookeeperServer; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } + @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } public TestProperties setFeedConcurrency(double feedConcurrency) { this.feedConcurrency = feedConcurrency; @@ -195,6 +197,11 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } + public TestProperties maxDeadBytesRatio(double ratio) { + maxDeadBytesRatio = ratio; + return this; + } + public static class Spec implements ConfigServerSpec { private final String hostName; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index d7df62d56cf..d1af71d3eb8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -66,6 +66,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; private final boolean useBucketExecutorForLidSpaceCompact; + private final double defaultMaxDeadBytesRatio; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -206,6 +207,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); useBucketExecutorForLidSpaceCompact = featureFlags.useBucketExecutorForLidSpaceCompact(); + defaultMaxDeadBytesRatio = featureFlags.maxDeadBytesRatio(); } public void setVisibilityDelay(double delay) { @@ -376,6 +378,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> .configid(getConfigId()) .visibilitydelay(visibilityDelay) .global(globalDocType); + ddbB.allocation.max_dead_bytes_ratio(defaultMaxDeadBytesRatio); if (hasIndexingModeStreaming(type)) { hasAnyNonIndexedCluster = true; diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 095434c8e04..7a2e9ad8da3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -998,6 +998,22 @@ public class ContentClusterTest extends ContentBaseTest { assertTrue(resolveThreePhaseUpdateConfigWithFeatureFlag(true)); } + private double resolveMaxDeadBytesRatio(double maxDeadBytesRatio) { + VespaModel model = createEnd2EndOneNode(new TestProperties().maxDeadBytesRatio(maxDeadBytesRatio)); + ContentCluster cc = model.getContentClusters().get("storage"); + ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); + cc.getSearch().getConfig(protonBuilder); + ProtonConfig protonConfig = new ProtonConfig(protonBuilder); + assertEquals(1, protonConfig.documentdb().size()); + return protonConfig.documentdb(0).allocation().max_dead_bytes_ratio(); + } + + @Test + public void default_max_dead_bytes_ratio_config_controlled_by_properties() { + assertEquals(0.2, resolveMaxDeadBytesRatio(0.2), 1e-5); + assertEquals(0.1, resolveMaxDeadBytesRatio(0.1), 1e-5); + } + void assertZookeeperServerImplementation(boolean reconfigurable, String expectedClassName) { VespaModel model = createEnd2EndOneNode( new TestProperties() diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index aa9ae65394f..7cbdacca838 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -164,6 +164,7 @@ public class ModelContextImpl implements ModelContext { private final boolean enableZstdCompressionAccessLog; private final boolean useBucketExecutorForLidSpaceCompact; private final boolean enableFeedBlockInDistributor; + private final double maxDeadBytesRatio; public FeatureFlags(FlagSource source, ApplicationId appId) { this.reindexerWindowSizeIncrement = flagValue(source, appId, Flags.REINDEXER_WINDOW_SIZE_INCREMENT); @@ -183,6 +184,7 @@ public class ModelContextImpl implements ModelContext { this.enableZstdCompressionAccessLog = flagValue(source, appId, Flags.ENABLE_ZSTD_COMPRESSION_ACCESS_LOG); this.useBucketExecutorForLidSpaceCompact = flagValue(source, appId, Flags.USE_BUCKET_EXECUTOR_FOR_LID_SPACE_COMPACT); this.enableFeedBlockInDistributor = flagValue(source, appId, Flags.ENABLE_FEED_BLOCK_IN_DISTRIBUTOR); + this.maxDeadBytesRatio = flagValue(source, appId, Flags.MAX_DEAD_BYTES_RATIO); } @Override public double reindexerWindowSizeIncrement() { return reindexerWindowSizeIncrement; } @@ -202,6 +204,7 @@ public class ModelContextImpl implements ModelContext { @Override public boolean enableZstdCompressionAccessLog() { return enableZstdCompressionAccessLog; } @Override public boolean useBucketExecutorForLidSpaceCompact() { return useBucketExecutorForLidSpaceCompact; } @Override public boolean enableFeedBlockInDistributor() { return enableFeedBlockInDistributor; } + @Override public double maxDeadBytesRatio() { return maxDeadBytesRatio; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index 075166e4759..b6962e38477 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -259,6 +259,12 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag MAX_DEAD_BYTES_RATIO = defineDoubleFlag( + "max-dead-bytes-ratio", 0.2, + List.of("baldersheim", "geirst","toregge"), "2021-02-03", "2022-02-03", + "max ratio of dead to used memory bytes in large data structures before compaction is attempted", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); /** WARNING: public for testing: All flags should be defined in {@link Flags}. */ public static UnboundBooleanFlag defineFeatureFlag(String flagId, boolean defaultValue, List<String> owners, |