diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-01-13 08:56:32 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2022-01-13 08:56:32 +0100 |
commit | bffdb7cd367c9641754f3ea3df5aef843b7b2759 (patch) | |
tree | f0ce5a22e3b46cb0e555282062c2ce67f6b7b761 /config-model | |
parent | 3578ae64641d6a3fb412d6ca1516e3cec0ac70e3 (diff) | |
parent | 888868b993af28be2ccb4bee98dbf82f4b189cca (diff) |
Merge branch 'master' into geirst/config-defaults-for-improved-merge-performance
Diffstat (limited to 'config-model')
10 files changed, 83 insertions, 102 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java index 3a3df4224b1..c148bb0e6e4 100644 --- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java +++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java @@ -67,16 +67,14 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea private boolean containerDumpHeapOnShutdownTimeout = false; private double containerShutdownTimeout = 50.0; private int distributorMergeBusyWait = 1; - private int docstoreCompressionLevel = 9; private int maxUnCommittedMemory = 123456; - private double diskBloatFactor = 0.25; private boolean distributorEnhancedMaintenanceScheduling = true; private boolean asyncApplyBucketDiff = true; private boolean unorderedMergeChaining = true; private List<String> zoneDnsSuffixes = List.of(); private int maxCompactBuffers = 1; private boolean failDeploymentWithInvalidJvmOptions = false; - private double tlsSizeFraction = 0.02; + private String persistenceAsyncThrottling = "UNLIMITED"; @Override public ModelContext.FeatureFlags featureFlags() { return this; } @Override public boolean multitenant() { return multitenant; } @@ -124,8 +122,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public double containerShutdownTimeout() { return containerShutdownTimeout; } @Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; } @Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; } - @Override public double diskBloatFactor() { return diskBloatFactor; } - @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; } @Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; } @Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; } @Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; } @@ -133,23 +129,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea @Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; } @Override public int maxCompactBuffers() { return maxCompactBuffers; } @Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; } - @Override public double tlsSizeFraction() { return tlsSizeFraction; } + @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; } public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) { this.maxUnCommittedMemory = maxUnCommittedMemory; return this; } - public TestProperties docstoreCompressionLevel(int docstoreCompressionLevel) { - this.docstoreCompressionLevel = docstoreCompressionLevel; - return this; - } - - public TestProperties diskBloatFactor(double diskBloatFactor) { - this.diskBloatFactor = diskBloatFactor; - return this; - } - public TestProperties containerDumpHeapOnShutdownTimeout(boolean value) { containerDumpHeapOnShutdownTimeout = value; return this; @@ -351,8 +337,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea return this; } - public TestProperties tlsSizeFraction(double tlsSizeFraction) { - this.tlsSizeFraction = tlsSizeFraction; + public TestProperties setPersistenceAsyncThrottling(String type) { + this.persistenceAsyncThrottling = type; return this; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 54d09bacfa9..d0cba617cfc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -43,6 +43,9 @@ import static java.util.stream.Collectors.toList; */ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> implements ProtonConfig.Producer, DispatchConfig.Producer { + private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3; + private static final double DEFAULT_DISK_BLOAT = 0.25; + private final boolean flushOnShutdown; private final Boolean syncTransactionLog; @@ -68,8 +71,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> private final int feedMasterTaskLimit; private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor; private final double defaultFeedConcurrency; - private final double defaultDiskBloatFactor; - private final int defaultDocStoreCompressionLevel; private final boolean forwardIssuesToQrs; private final int defaultMaxCompactBuffers; @@ -223,8 +224,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit(); this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor()); this.defaultFeedConcurrency = featureFlags.feedConcurrency(); - this.defaultDiskBloatFactor = featureFlags.diskBloatFactor(); - this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel(); this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors(); this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers(); } @@ -291,7 +290,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags().tlsSizeFraction()); + fractionOfMemoryReserved); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState.getDeployLogger()); @@ -423,10 +422,10 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> } else { builder.feeding.concurrency(defaultFeedConcurrency); } - builder.flush.memory.diskbloatfactor(defaultDiskBloatFactor); - builder.flush.memory.each.diskbloatfactor(defaultDiskBloatFactor); - builder.summary.log.chunk.compression.level(defaultDocStoreCompressionLevel); - builder.summary.log.compact.compression.level(defaultDocStoreCompressionLevel); + builder.flush.memory.diskbloatfactor(DEFAULT_DISK_BLOAT); + builder.flush.memory.each.diskbloatfactor(DEFAULT_DISK_BLOAT); + builder.summary.log.chunk.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); + builder.summary.log.compact.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL); builder.forward_issues(forwardIssuesToQrs); int numDocumentDbs = builder.documentdb.size(); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java index d7f6fb6c581..1f3a76b766e 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java @@ -46,6 +46,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { private final ContentCluster cluster; private final int reponseNumThreads; private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType; + private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType; private final boolean useAsyncMessageHandlingOnSchedule; private final boolean asyncApplyBucketDiff; @@ -57,11 +58,20 @@ public class FileStorProducer implements StorFilestorConfig.Producer { } } + private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) { + try { + return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType); + } catch (Throwable t) { + return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED; + } + } + public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) { this.numThreads = numThreads; this.cluster = parent; this.reponseNumThreads = featureFlags.defaultNumResponseThreads(); this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType()); + this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling()); useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule(); asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff(); } @@ -76,6 +86,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer { builder.response_sequencer_type(responseSequencerType); builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule); builder.async_apply_bucket_diff(asyncApplyBucketDiff); + builder.async_operation_throttler_type(asyncOperationThrottlerType); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index e321892b5ad..9b9a525ab29 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -17,6 +17,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY = 0.02; private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.04; private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08; + private final static double TLS_SIZE_FRACTION = 0.02; final static long MB = 1024 * 1024; public final static long GB = MB * 1024; // This is an approximate number base on observation of a node using 33G memory with 765M docs @@ -24,19 +25,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; - private final double tlsSizeFraction; // "Reserve" 0.5GB of memory for other processes running on the content node (config-proxy, metrics-proxy). public static final double reservedMemoryGb = 0.5; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - double fractionOfMemoryReserved, - double tlsSizeFraction) { + double fractionOfMemoryReserved) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.tlsSizeFraction = tlsSizeFraction; } @Override @@ -93,7 +91,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) { - long tlsSizeBytes = (long) ((resources.diskGb() * tlsSizeFraction) * GB); + long tlsSizeBytes = (long) ((resources.diskGb() * TLS_SIZE_FRACTION) * GB); tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB)); builder.maxtlssize(tlsSizeBytes); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 31513a273b2..28d1fbe72ef 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -60,7 +60,7 @@ public class SearchNode extends AbstractService implements private final boolean isHostedVespa; private final boolean flushOnShutdown; - private NodeSpec nodeSpec; + private final NodeSpec nodeSpec; private int distributionKey; private final String clusterName; private TransactionLogServer tls; @@ -68,7 +68,6 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; - private final double tlsSizeFraction; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { @@ -97,8 +96,7 @@ public class SearchNode extends AbstractService implements @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved, - deployState.featureFlags().tlsSizeFraction()); + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved); } } @@ -106,16 +104,16 @@ public class SearchNode extends AbstractService implements public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, double tlsSizeFraction) { + double fractionOfMemoryReserved) { return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, double tlsSizeFraction) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction); + double fractionOfMemoryReserved) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; setPropertiesElastic(clusterName, distributionKey); @@ -123,12 +121,11 @@ public class SearchNode extends AbstractService implements private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, double tlsSizeFraction) { + double fractionOfMemoryReserved) { super(parent, name); setOmpNumThreads(1); this.isHostedVespa = isHostedVespa; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.tlsSizeFraction = tlsSizeFraction; this.nodeSpec = nodeSpec; this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; @@ -282,7 +279,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved, tlsSizeFraction); + fractionOfMemoryReserved); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java index 4591726d1f9..87a962339e9 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java @@ -45,7 +45,6 @@ import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.OptionalDouble; import java.util.OptionalInt; import static org.junit.Assert.assertEquals; @@ -1046,11 +1045,9 @@ public class ContentClusterTest extends ContentBaseTest { assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7))); } - private long resolveMaxTLSSize(OptionalDouble tlsSizeFraction, Optional<Flavor> flavor) throws Exception { + private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception { TestProperties testProperties = new TestProperties(); - if (tlsSizeFraction.isPresent()) { - testProperties.tlsSizeFraction(tlsSizeFraction.getAsDouble()); - } + ContentCluster cc = createOneNodeCluster(testProperties, flavor); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); cc.getSearch().getSearchNodes().get(0).getConfig(protonBuilder); @@ -1058,13 +1055,10 @@ public class ContentClusterTest extends ContentBaseTest { return protonConfig.flush().memory().maxtlssize(); } @Test - public void default_max_tls_size_controlled_by_properties() throws Exception { + public void verifyt_max_tls_size() throws Exception { var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100))); - assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.empty())); - assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.empty())); - assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.of(flavor))); - assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.of(flavor))); - assertEquals(3221225472L, resolveMaxTLSSize(OptionalDouble.of(0.03), Optional.of(flavor))); + assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty())); + assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor))); } void assertZookeeperServerImplementation(String expectedClassName, diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java index 51badae2746..68e722f45d3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java @@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue; */ public class ContentSchemaClusterTest { - private static double EPSILON = 0.000001; + private static final double EPSILON = 0.000001; private static ContentCluster createClusterWithOneDocumentType() throws Exception { return createCluster(new ContentClusterBuilder().getXml()); @@ -261,27 +261,16 @@ public class ContentSchemaClusterTest { } @Test - public void verifyControlOfDocStoreCompression() throws Exception { + public void verifyDefaultDocStoreCompression() throws Exception { ProtonConfig cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); - assertEquals(9, cfg.summary().log().chunk().compression().level()); - assertEquals(9, cfg.summary().log().compact().compression().level()); - - cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), - new DeployState.Builder().properties(new TestProperties().docstoreCompressionLevel(3)))); assertEquals(3, cfg.summary().log().chunk().compression().level()); assertEquals(3, cfg.summary().log().compact().compression().level()); } @Test - public void verifyControlOfDiskBloatFactor() throws Exception { + public void verifyDefaultDiskBloatFactor() throws Exception { var defaultCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml())); assertEquals(0.25, defaultCfg.flush().memory().diskbloatfactor(), EPSILON); assertEquals(0.25, defaultCfg.flush().memory().each().diskbloatfactor(), EPSILON); - - var controlledCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(), - new DeployState.Builder().properties(new TestProperties().diskBloatFactor(0.31)) - )); - assertEquals(0.31, controlledCfg.flush().memory().diskbloatfactor(), EPSILON); - assertEquals(0.31, controlledCfg.flush().memory().each().diskbloatfactor(), EPSILON); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java index c521d24d22f..cf877d3bf88 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java @@ -306,6 +306,25 @@ public class StorageClusterTest { } @Test + public void persistence_async_throttle_config_defaults_to_unlimited() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties())); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + } + + @Test + public void persistence_async_throttle_config_is_derived_from_flag() { + var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + + config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type()); + + // Invalid enum values fall back to the default + config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS"))); + assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type()); + } + + @Test public void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() { StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder(); parse(cluster("bees", "")).getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 18ad9a4def9..5571ead11ce 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -42,7 +42,7 @@ public class NodeResourcesTuningTest { assertEquals(0.5, reservedMemoryGb, delta); } - private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb, int redundancy, int searchableCopies) { + private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); for (Pair<String, String> sdMode : sdAndMode) { builder.documentdb.add(new ProtonConfig.Documentdb.Builder() @@ -53,8 +53,8 @@ public class NodeResourcesTuningTest { return configFromMemorySetting(gb, builder); } - private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) { - ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb, redundancy, searchablecopies); + private void verify_that_initial_numdocs_is_dependent_of_mode() { + ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb); assertEquals(3, cfg.documentdb().size()); assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); assertEquals("a", cfg.documentdb(0).inputdoctypename()); @@ -66,10 +66,8 @@ public class NodeResourcesTuningTest { @Test public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { - verify_that_initial_numdocs_is_dependent_of_mode(2,0); - verify_that_initial_numdocs_is_dependent_of_mode(1,1); - verify_that_initial_numdocs_is_dependent_of_mode(3, 2); - verify_that_initial_numdocs_is_dependent_of_mode(3, 3); + verify_that_initial_numdocs_is_dependent_of_mode(); + } @Test @@ -134,13 +132,11 @@ public class NodeResourcesTuningTest { @Test public void require_that_flush_strategy_tls_size_is_set_based_on_available_disk() { - assertFlushStrategyTlsSize(2 * GB, 10, 0.05); - assertFlushStrategyTlsSize(7 * GB, 100, 0.07); - assertFlushStrategyTlsSize(5 * GB, 100, 0.05); - assertFlushStrategyTlsSize(35 * GB, 500, 0.07); - assertFlushStrategyTlsSize(84 * GB, 1200, 0.07); - assertFlushStrategyTlsSize(100 * GB, 1720, 0.07); - assertFlushStrategyTlsSize(100 * GB, 24000, 0.07); + assertFlushStrategyTlsSize(2 * GB, 10); + assertFlushStrategyTlsSize(2 * GB, 100); + assertFlushStrategyTlsSize(10 * GB, 500); + assertFlushStrategyTlsSize(24 * GB, 1200); + assertFlushStrategyTlsSize(100 * GB, 24000); } @Test @@ -181,8 +177,8 @@ public class NodeResourcesTuningTest { assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().each().maxmemory()); } - private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb, double tlsSizeFraction) { - assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb, tlsSizeFraction).flush().memory().maxtlssize()); + private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { + assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb).flush().memory().maxtlssize()); } private static void assertSummaryReadIo(ProtonConfig.Summary.Read.Io.Enum expValue, boolean fastDisk) { @@ -197,23 +193,16 @@ public class NodeResourcesTuningTest { assertEquals(sharedDisk, configFromEnvironmentType(docker).hwinfo().disk().shared()); } - private static void assertWriteFilter(double expMemoryLimit, int memoryGb) { - assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, 0).writefilter().memorylimit(), delta); - } - private static ProtonConfig configFromDiskSetting(boolean fastDisk) { return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk)); } private static ProtonConfig configFromDiskSetting(int diskGb) { - return configFromDiskSetting(diskGb, 0.07); - } - private static ProtonConfig configFromDiskSetting(int diskGb, double tlsSizeFraction) { - return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0, tlsSizeFraction); + return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0); } private static ProtonConfig configFromMemorySetting(double memoryGb, double fractionOfMemoryReserved) { - return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved, 0.07); + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved); } private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { @@ -239,26 +228,26 @@ public class NodeResourcesTuningTest { return getConfig(flavorBuilder, new ProtonConfig.Builder()); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { - return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved, tlsSizeFraction); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved) { + return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { return getConfig(flavorBuilder, protonBuilder,1); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) { - return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved, tlsSizeFraction); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) { + return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, 0.07); + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, double fractionOfMemoryReserved, double tlsSizeFraction) { + int numThreadsPerSearch, double fractionOfMemoryReserved) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved, tlsSizeFraction); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java index d000f83d6cd..226045f4d8a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java @@ -51,8 +51,7 @@ public class SchemaNodeTest { private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - Optional.empty(), Optional.empty(), isHosted, 0.0, - root.getDeployState().featureFlags().tlsSizeFraction()); + Optional.empty(), Optional.empty(), isHosted, 0.0); } private static SearchNode createSearchNode(MockRoot root) { |