diff options
5 files changed, 61 insertions, 1 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index a6f8d510c0d..ce4e5f0c01f 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -50,7 +50,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean useAsyncMessageHandlingOnSchedule = false; private double feedConcurrency = 0.5; private boolean enableFeedBlockInDistributor = true; - private boolean enforceRankProfileInheritance = true; private int maxActivationInhibitedOutOfSyncGroups = 0; private List<TenantSecretStore> tenantSecretStores = Collections.emptyList(); private String jvmOmitStackTraceInFastThrowOption; @@ -66,6 +65,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; private int distributorMergeBusyWait = 10; + private int docstoreCompressionLevel = 9; + private double diskBloatFactor = 0.2; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -111,7 +112,18 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } + @Override public double diskBloatFactor() { return diskBloatFactor; } + @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } + public TestProperties docstoreCompressionLevel(int docstoreCompressionLevel) { + this.docstoreCompressionLevel = docstoreCompressionLevel; + return this; + } + + public TestProperties diskBloatFactor(double diskBloatFactor) { + this.diskBloatFactor = diskBloatFactor; + return this; + } public TestProperties containerDumpHeapOnShutdownTimeout(boolean value) { containerDumpHeapOnShutdownTimeout = value; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index ba4daa84fe6..51c5d85e300 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -65,6 +65,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private Optional<ResourceLimits> resourceLimits = Optional.empty(); private final ProtonConfig.Indexing.Optimize.Enum feedSequencerType; private final double defaultFeedConcurrency; + private final double defaultDiskBloatFactor; + private final int defaultDocStoreCompressionLevel; /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ private final boolean combined; @@ -209,6 +211,8 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.combined = combined; feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType()); defaultFeedConcurrency = featureFlags.feedConcurrency(); + defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); + defaultDiskBloatFactor = featureFlags.diskBloatFactor(); } public void setVisibilityDelay(double delay) { @@ -401,6 +405,9 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.feeding.concurrency(defaultFeedConcurrency); } + builder.flush.memory.diskbloatfactor(defaultDiskBloatFactor); + builder.flush.memory.each.diskbloatfactor(defaultDiskBloatFactor); + builder.summary.log.chunk.compression.level(defaultDocStoreCompressionLevel); int numDocumentDbs = builder.documentdb.size(); builder.initialize(new ProtonConfig.Initialize.Builder().threads(numDocumentDbs + 1)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java index c5b2b71a6b4..1ab41c8a404 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSearchClusterTest.java @@ -259,4 +259,25 @@ public class ContentSearchClusterTest { assertTrue(getTlsConfig(createCluster(new ContentClusterBuilder().syncTransactionLog(true).getXml())).usefsync()); assertFalse(getTlsConfig(createCluster(new ContentClusterBuilder().syncTransactionLog(false).getXml())).usefsync()); } + + @Test + public void verifyControlOfDocStoreCompression() throws Exception { + assertEquals(9, getProtonConfig(createCluster(new ContentClusterBuilder().getXml())).summary().log().chunk().compression().level()); + assertEquals(3, getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), + new DeployState.Builder().properties(new TestProperties().docstoreCompressionLevel(3)) + )).summary().log().chunk().compression().level()); + } + + @Test + public void verifyControlOfDiskBloatFactor() throws Exception { + var defaultCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); + assertEquals(0.2, defaultCfg.flush().memory().diskbloatfactor(), EPSILON); + assertEquals(0.2, defaultCfg.flush().memory().each().diskbloatfactor(), EPSILON); + + var controlledCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), + new DeployState.Builder().properties(new TestProperties().diskBloatFactor(0.31)) + )); + assertEquals(0.31, controlledCfg.flush().memory().diskbloatfactor(), EPSILON); + assertEquals(0.31, controlledCfg.flush().memory().each().diskbloatfactor(), EPSILON); + } } diff --git a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java index 4f178d29e59..b25bbf1a5bb 100644 --- a/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java +++ b/configserver/src/main/java/com/yahoo/vespa/config/server/deploy/ModelContextImpl.java @@ -192,6 +192,8 @@ public class ModelContextImpl implements ModelContext { private final double containerShutdownTimeout; private final int maxConnectionLifeInHosted; private final int distributorMergeBusyWait; + private final int docstoreCompressionLevel; + private final double diskBloatFactor; public FeatureFlags(FlagSource source, ApplicationId appId) { this.defaultTermwiseLimit = flagValue(source, appId, Flags.DEFAULT_TERM_WISE_LIMIT); @@ -222,6 +224,8 @@ public class ModelContextImpl implements ModelContext { this.containerShutdownTimeout = flagValue(source, appId,Flags.CONTAINER_SHUTDOWN_TIMEOUT); this.maxConnectionLifeInHosted = flagValue(source, appId, Flags.MAX_CONNECTION_LIFE_IN_HOSTED); this.distributorMergeBusyWait = flagValue(source, appId, Flags.DISTRIBUTOR_MERGE_BUSY_WAIT); + this.docstoreCompressionLevel = flagValue(source, appId, Flags.DOCSTORE_COMPRESSION_LEVEL); + this.diskBloatFactor = flagValue(source, appId, Flags.DISK_BLOAT_FACTOR); } @Override public double defaultTermwiseLimit() { return defaultTermwiseLimit; } @@ -254,6 +258,8 @@ public class ModelContextImpl implements ModelContext { @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int maxConnectionLifeInHosted() { return maxConnectionLifeInHosted; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } + @Override public double diskBloatFactor() { return diskBloatFactor; } + @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } private static <V> V flagValue(FlagSource source, ApplicationId appId, UnboundFlag<? extends V, ?, ?> flag) { return flag.bindTo(source) diff --git a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java index d576bb335ab..c8c55faa2c7 100644 --- a/flags/src/main/java/com/yahoo/vespa/flags/Flags.java +++ b/flags/src/main/java/com/yahoo/vespa/flags/Flags.java @@ -128,6 +128,20 @@ public class Flags { "Takes effect at redeployment", ZONE_ID, APPLICATION_ID); + public static final UnboundDoubleFlag DISK_BLOAT_FACTOR = defineDoubleFlag( + "disk-bloat-factor", 0.2, + List.of("baldersheim"), "2021-10-08", "2022-01-01", + "Amount of bloat allowed before compacting file", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + + public static final UnboundIntFlag DOCSTORE_COMPRESSION_LEVEL = defineIntFlag( + "docstore-compression-level", 9, + List.of("baldersheim"), "2021-10-08", "2022-01-01", + "Default compression level used for document store", + "Takes effect at redeployment", + ZONE_ID, APPLICATION_ID); + public static final UnboundIntFlag NUM_DEPLOY_HELPER_THREADS = defineIntFlag( "num-model-builder-threads", -1, List.of("balder"), "2021-09-09", "2021-11-01", |