aboutsummaryrefslogtreecommitdiffstats
path: root/config-model
diff options
context:
space:
mode:
authorHenning Baldersheim <balder@yahoo-inc.com>2022-01-13 08:56:32 +0100
committerGitHub <noreply@github.com>2022-01-13 08:56:32 +0100
commitbffdb7cd367c9641754f3ea3df5aef843b7b2759 (patch)
treef0ce5a22e3b46cb0e555282062c2ce67f6b7b761 /config-model
parent3578ae64641d6a3fb412d6ca1516e3cec0ac70e3 (diff)
parent888868b993af28be2ccb4bee98dbf82f4b189cca (diff)
Merge branch 'master' into geirst/config-defaults-for-improved-merge-performance
Diffstat (limited to 'config-model')
-rw-r--r--config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java22
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java17
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java11
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java8
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java19
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java16
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java17
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java19
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java53
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java3
10 files changed, 83 insertions, 102 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
index 3a3df4224b1..c148bb0e6e4 100644
--- a/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
+++ b/config-model/src/main/java/com/yahoo/config/model/deploy/TestProperties.java
@@ -67,16 +67,14 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
private boolean containerDumpHeapOnShutdownTimeout = false;
private double containerShutdownTimeout = 50.0;
private int distributorMergeBusyWait = 1;
- private int docstoreCompressionLevel = 9;
private int maxUnCommittedMemory = 123456;
- private double diskBloatFactor = 0.25;
private boolean distributorEnhancedMaintenanceScheduling = true;
private boolean asyncApplyBucketDiff = true;
private boolean unorderedMergeChaining = true;
private List<String> zoneDnsSuffixes = List.of();
private int maxCompactBuffers = 1;
private boolean failDeploymentWithInvalidJvmOptions = false;
- private double tlsSizeFraction = 0.02;
+ private String persistenceAsyncThrottling = "UNLIMITED";
@Override public ModelContext.FeatureFlags featureFlags() { return this; }
@Override public boolean multitenant() { return multitenant; }
@@ -124,8 +122,6 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
@Override public double containerShutdownTimeout() { return containerShutdownTimeout; }
@Override public boolean containerDumpHeapOnShutdownTimeout() { return containerDumpHeapOnShutdownTimeout; }
@Override public int distributorMergeBusyWait() { return distributorMergeBusyWait; }
- @Override public double diskBloatFactor() { return diskBloatFactor; }
- @Override public int docstoreCompressionLevel() { return docstoreCompressionLevel; }
@Override public boolean distributorEnhancedMaintenanceScheduling() { return distributorEnhancedMaintenanceScheduling; }
@Override public int maxUnCommittedMemory() { return maxUnCommittedMemory; }
@Override public boolean asyncApplyBucketDiff() { return asyncApplyBucketDiff; }
@@ -133,23 +129,13 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
@Override public List<String> zoneDnsSuffixes() { return zoneDnsSuffixes; }
@Override public int maxCompactBuffers() { return maxCompactBuffers; }
@Override public boolean failDeploymentWithInvalidJvmOptions() { return failDeploymentWithInvalidJvmOptions; }
- @Override public double tlsSizeFraction() { return tlsSizeFraction; }
+ @Override public String persistenceAsyncThrottling() { return persistenceAsyncThrottling; }
public TestProperties maxUnCommittedMemory(int maxUnCommittedMemory) {
this.maxUnCommittedMemory = maxUnCommittedMemory;
return this;
}
- public TestProperties docstoreCompressionLevel(int docstoreCompressionLevel) {
- this.docstoreCompressionLevel = docstoreCompressionLevel;
- return this;
- }
-
- public TestProperties diskBloatFactor(double diskBloatFactor) {
- this.diskBloatFactor = diskBloatFactor;
- return this;
- }
-
public TestProperties containerDumpHeapOnShutdownTimeout(boolean value) {
containerDumpHeapOnShutdownTimeout = value;
return this;
@@ -351,8 +337,8 @@ public class TestProperties implements ModelContext.Properties, ModelContext.Fea
return this;
}
- public TestProperties tlsSizeFraction(double tlsSizeFraction) {
- this.tlsSizeFraction = tlsSizeFraction;
+ public TestProperties setPersistenceAsyncThrottling(String type) {
+ this.persistenceAsyncThrottling = type;
return this;
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index 54d09bacfa9..d0cba617cfc 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -43,6 +43,9 @@ import static java.util.stream.Collectors.toList;
*/
public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster> implements ProtonConfig.Producer, DispatchConfig.Producer {
+ private static final int DEFAULT_DOC_STORE_COMPRESSION_LEVEL = 3;
+ private static final double DEFAULT_DISK_BLOAT = 0.25;
+
private final boolean flushOnShutdown;
private final Boolean syncTransactionLog;
@@ -68,8 +71,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
private final int feedMasterTaskLimit;
private final ProtonConfig.Feeding.Shared_field_writer_executor.Enum sharedFieldWriterExecutor;
private final double defaultFeedConcurrency;
- private final double defaultDiskBloatFactor;
- private final int defaultDocStoreCompressionLevel;
private final boolean forwardIssuesToQrs;
private final int defaultMaxCompactBuffers;
@@ -223,8 +224,6 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit();
this.sharedFieldWriterExecutor = convertSharedFieldWriterExecutor(featureFlags.sharedFieldWriterExecutor());
this.defaultFeedConcurrency = featureFlags.feedConcurrency();
- this.defaultDiskBloatFactor = featureFlags.diskBloatFactor();
- this.defaultDocStoreCompressionLevel = featureFlags.docstoreCompressionLevel();
this.forwardIssuesToQrs = featureFlags.forwardIssuesAsErrors();
this.defaultMaxCompactBuffers = featureFlags.maxCompactBuffers();
}
@@ -291,7 +290,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
if (element == null) {
searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec,
clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(),
- fractionOfMemoryReserved, deployState.featureFlags().tlsSizeFraction());
+ fractionOfMemoryReserved);
searchNode.setHostResource(node.getHostResource());
searchNode.initService(deployState.getDeployLogger());
@@ -423,10 +422,10 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
} else {
builder.feeding.concurrency(defaultFeedConcurrency);
}
- builder.flush.memory.diskbloatfactor(defaultDiskBloatFactor);
- builder.flush.memory.each.diskbloatfactor(defaultDiskBloatFactor);
- builder.summary.log.chunk.compression.level(defaultDocStoreCompressionLevel);
- builder.summary.log.compact.compression.level(defaultDocStoreCompressionLevel);
+ builder.flush.memory.diskbloatfactor(DEFAULT_DISK_BLOAT);
+ builder.flush.memory.each.diskbloatfactor(DEFAULT_DISK_BLOAT);
+ builder.summary.log.chunk.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL);
+ builder.summary.log.compact.compression.level(DEFAULT_DOC_STORE_COMPRESSION_LEVEL);
builder.forward_issues(forwardIssuesToQrs);
int numDocumentDbs = builder.documentdb.size();
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
index d7f6fb6c581..1f3a76b766e 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/FileStorProducer.java
@@ -46,6 +46,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
private final ContentCluster cluster;
private final int reponseNumThreads;
private final StorFilestorConfig.Response_sequencer_type.Enum responseSequencerType;
+ private final StorFilestorConfig.Async_operation_throttler_type.Enum asyncOperationThrottlerType;
private final boolean useAsyncMessageHandlingOnSchedule;
private final boolean asyncApplyBucketDiff;
@@ -57,11 +58,20 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
}
}
+ private static StorFilestorConfig.Async_operation_throttler_type.Enum toAsyncOperationThrottlerType(String throttlerType) {
+ try {
+ return StorFilestorConfig.Async_operation_throttler_type.Enum.valueOf(throttlerType);
+ } catch (Throwable t) {
+ return StorFilestorConfig.Async_operation_throttler_type.UNLIMITED;
+ }
+ }
+
public FileStorProducer(ModelContext.FeatureFlags featureFlags, ContentCluster parent, Integer numThreads) {
this.numThreads = numThreads;
this.cluster = parent;
this.reponseNumThreads = featureFlags.defaultNumResponseThreads();
this.responseSequencerType = convertResponseSequencerType(featureFlags.responseSequencerType());
+ this.asyncOperationThrottlerType = toAsyncOperationThrottlerType(featureFlags.persistenceAsyncThrottling());
useAsyncMessageHandlingOnSchedule = featureFlags.useAsyncMessageHandlingOnSchedule();
asyncApplyBucketDiff = featureFlags.asyncApplyBucketDiff();
}
@@ -76,6 +86,7 @@ public class FileStorProducer implements StorFilestorConfig.Producer {
builder.response_sequencer_type(responseSequencerType);
builder.use_async_message_handling_on_schedule(useAsyncMessageHandlingOnSchedule);
builder.async_apply_bucket_diff(asyncApplyBucketDiff);
+ builder.async_operation_throttler_type(asyncOperationThrottlerType);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
index e321892b5ad..9b9a525ab29 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
@@ -17,6 +17,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
private final static double SUMMARY_FILE_SIZE_AS_FRACTION_OF_MEMORY = 0.02;
private final static double SUMMARY_CACHE_SIZE_AS_FRACTION_OF_MEMORY = 0.04;
private final static double MEMORY_GAIN_AS_FRACTION_OF_MEMORY = 0.08;
+ private final static double TLS_SIZE_FRACTION = 0.02;
final static long MB = 1024 * 1024;
public final static long GB = MB * 1024;
// This is an approximate number base on observation of a node using 33G memory with 765M docs
@@ -24,19 +25,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
private final NodeResources resources;
private final int threadsPerSearch;
private final double fractionOfMemoryReserved;
- private final double tlsSizeFraction;
// "Reserve" 0.5GB of memory for other processes running on the content node (config-proxy, metrics-proxy).
public static final double reservedMemoryGb = 0.5;
public NodeResourcesTuning(NodeResources resources,
int threadsPerSearch,
- double fractionOfMemoryReserved,
- double tlsSizeFraction) {
+ double fractionOfMemoryReserved) {
this.resources = resources;
this.threadsPerSearch = threadsPerSearch;
this.fractionOfMemoryReserved = fractionOfMemoryReserved;
- this.tlsSizeFraction = tlsSizeFraction;
}
@Override
@@ -93,7 +91,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
}
private void tuneFlushStrategyTlsSize(ProtonConfig.Flush.Memory.Builder builder) {
- long tlsSizeBytes = (long) ((resources.diskGb() * tlsSizeFraction) * GB);
+ long tlsSizeBytes = (long) ((resources.diskGb() * TLS_SIZE_FRACTION) * GB);
tlsSizeBytes = max(2*GB, min(tlsSizeBytes, 100 * GB));
builder.maxtlssize(tlsSizeBytes);
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 31513a273b2..28d1fbe72ef 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -60,7 +60,7 @@ public class SearchNode extends AbstractService implements
private final boolean isHostedVespa;
private final boolean flushOnShutdown;
- private NodeSpec nodeSpec;
+ private final NodeSpec nodeSpec;
private int distributionKey;
private final String clusterName;
private TransactionLogServer tls;
@@ -68,7 +68,6 @@ public class SearchNode extends AbstractService implements
private final Optional<Tuning> tuning;
private final Optional<ResourceLimits> resourceLimits;
private final double fractionOfMemoryReserved;
- private final double tlsSizeFraction;
public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> {
@@ -97,8 +96,7 @@ public class SearchNode extends AbstractService implements
@Override
protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) {
return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode,
- flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved,
- deployState.featureFlags().tlsSizeFraction());
+ flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved);
}
}
@@ -106,16 +104,16 @@ public class SearchNode extends AbstractService implements
public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec,
String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown,
Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
- double fractionOfMemoryReserved, double tlsSizeFraction) {
+ double fractionOfMemoryReserved) {
return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown,
- tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction);
+ tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved);
}
private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec,
String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown,
Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
- double fractionOfMemoryReserved, double tlsSizeFraction) {
- this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, tlsSizeFraction);
+ double fractionOfMemoryReserved) {
+ this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved);
this.distributionKey = distributionKey;
this.serviceLayerService = serviceLayerService;
setPropertiesElastic(clusterName, distributionKey);
@@ -123,12 +121,11 @@ public class SearchNode extends AbstractService implements
private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName,
boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
- double fractionOfMemoryReserved, double tlsSizeFraction) {
+ double fractionOfMemoryReserved) {
super(parent, name);
setOmpNumThreads(1);
this.isHostedVespa = isHostedVespa;
this.fractionOfMemoryReserved = fractionOfMemoryReserved;
- this.tlsSizeFraction = tlsSizeFraction;
this.nodeSpec = nodeSpec;
this.clusterName = clusterName;
this.flushOnShutdown = flushOnShutdown;
@@ -282,7 +279,7 @@ public class SearchNode extends AbstractService implements
if (nodeResources.isPresent()) {
var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(),
tuning.map(Tuning::threadsPerSearch).orElse(1),
- fractionOfMemoryReserved, tlsSizeFraction);
+ fractionOfMemoryReserved);
nodeResourcesTuning.getConfig(builder);
tuning.ifPresent(t -> t.getConfig(builder));
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
index 4591726d1f9..87a962339e9 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentClusterTest.java
@@ -45,7 +45,6 @@ import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.OptionalDouble;
import java.util.OptionalInt;
import static org.junit.Assert.assertEquals;
@@ -1046,11 +1045,9 @@ public class ContentClusterTest extends ContentBaseTest {
assertEquals(7, resolveMaxCompactBuffers(OptionalInt.of(7)));
}
- private long resolveMaxTLSSize(OptionalDouble tlsSizeFraction, Optional<Flavor> flavor) throws Exception {
+ private long resolveMaxTLSSize(Optional<Flavor> flavor) throws Exception {
TestProperties testProperties = new TestProperties();
- if (tlsSizeFraction.isPresent()) {
- testProperties.tlsSizeFraction(tlsSizeFraction.getAsDouble());
- }
+
ContentCluster cc = createOneNodeCluster(testProperties, flavor);
ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder();
cc.getSearch().getSearchNodes().get(0).getConfig(protonBuilder);
@@ -1058,13 +1055,10 @@ public class ContentClusterTest extends ContentBaseTest {
return protonConfig.flush().memory().maxtlssize();
}
@Test
- public void default_max_tls_size_controlled_by_properties() throws Exception {
+ public void verifyt_max_tls_size() throws Exception {
var flavor = new Flavor(new FlavorsConfig.Flavor(new FlavorsConfig.Flavor.Builder().name("test").minDiskAvailableGb(100)));
- assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.empty()));
- assertEquals(21474836480L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.empty()));
- assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.empty(), Optional.of(flavor)));
- assertEquals(2147483648L, resolveMaxTLSSize(OptionalDouble.of(0.02), Optional.of(flavor)));
- assertEquals(3221225472L, resolveMaxTLSSize(OptionalDouble.of(0.03), Optional.of(flavor)));
+ assertEquals(21474836480L, resolveMaxTLSSize(Optional.empty()));
+ assertEquals(2147483648L, resolveMaxTLSSize(Optional.of(flavor)));
}
void assertZookeeperServerImplementation(String expectedClassName,
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java
index 51badae2746..68e722f45d3 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/ContentSchemaClusterTest.java
@@ -33,7 +33,7 @@ import static org.junit.Assert.assertTrue;
*/
public class ContentSchemaClusterTest {
- private static double EPSILON = 0.000001;
+ private static final double EPSILON = 0.000001;
private static ContentCluster createClusterWithOneDocumentType() throws Exception {
return createCluster(new ContentClusterBuilder().getXml());
@@ -261,27 +261,16 @@ public class ContentSchemaClusterTest {
}
@Test
- public void verifyControlOfDocStoreCompression() throws Exception {
+ public void verifyDefaultDocStoreCompression() throws Exception {
ProtonConfig cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml()));
- assertEquals(9, cfg.summary().log().chunk().compression().level());
- assertEquals(9, cfg.summary().log().compact().compression().level());
-
- cfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(),
- new DeployState.Builder().properties(new TestProperties().docstoreCompressionLevel(3))));
assertEquals(3, cfg.summary().log().chunk().compression().level());
assertEquals(3, cfg.summary().log().compact().compression().level());
}
@Test
- public void verifyControlOfDiskBloatFactor() throws Exception {
+ public void verifyDefaultDiskBloatFactor() throws Exception {
var defaultCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml()));
assertEquals(0.25, defaultCfg.flush().memory().diskbloatfactor(), EPSILON);
assertEquals(0.25, defaultCfg.flush().memory().each().diskbloatfactor(), EPSILON);
-
- var controlledCfg = getProtonConfig(createCluster(new ContentClusterBuilder().getXml(),
- new DeployState.Builder().properties(new TestProperties().diskBloatFactor(0.31))
- ));
- assertEquals(0.31, controlledCfg.flush().memory().diskbloatfactor(), EPSILON);
- assertEquals(0.31, controlledCfg.flush().memory().each().diskbloatfactor(), EPSILON);
}
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
index c521d24d22f..cf877d3bf88 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/content/StorageClusterTest.java
@@ -306,6 +306,25 @@ public class StorageClusterTest {
}
@Test
+ public void persistence_async_throttle_config_defaults_to_unlimited() {
+ var config = filestorConfigFromProducer(simpleCluster(new TestProperties()));
+ assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type());
+ }
+
+ @Test
+ public void persistence_async_throttle_config_is_derived_from_flag() {
+ var config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("UNLIMITED")));
+ assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type());
+
+ config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("DYNAMIC")));
+ assertEquals(StorFilestorConfig.Async_operation_throttler_type.DYNAMIC, config.async_operation_throttler_type());
+
+ // Invalid enum values fall back to the default
+ config = filestorConfigFromProducer(simpleCluster(new TestProperties().setPersistenceAsyncThrottling("BANANAS")));
+ assertEquals(StorFilestorConfig.Async_operation_throttler_type.UNLIMITED, config.async_operation_throttler_type());
+ }
+
+ @Test
public void integrity_checker_explicitly_disabled_when_not_running_with_vds_provider() {
StorIntegritycheckerConfig.Builder builder = new StorIntegritycheckerConfig.Builder();
parse(cluster("bees", "")).getConfig(builder);
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
index 18ad9a4def9..5571ead11ce 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
@@ -42,7 +42,7 @@ public class NodeResourcesTuningTest {
assertEquals(0.5, reservedMemoryGb, delta);
}
- private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb, int redundancy, int searchableCopies) {
+ private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb) {
ProtonConfig.Builder builder = new ProtonConfig.Builder();
for (Pair<String, String> sdMode : sdAndMode) {
builder.documentdb.add(new ProtonConfig.Documentdb.Builder()
@@ -53,8 +53,8 @@ public class NodeResourcesTuningTest {
return configFromMemorySetting(gb, builder);
}
- private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) {
- ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb, redundancy, searchablecopies);
+ private void verify_that_initial_numdocs_is_dependent_of_mode() {
+ ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + reservedMemoryGb);
assertEquals(3, cfg.documentdb().size());
assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs());
assertEquals("a", cfg.documentdb(0).inputdoctypename());
@@ -66,10 +66,8 @@ public class NodeResourcesTuningTest {
@Test
public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() {
- verify_that_initial_numdocs_is_dependent_of_mode(2,0);
- verify_that_initial_numdocs_is_dependent_of_mode(1,1);
- verify_that_initial_numdocs_is_dependent_of_mode(3, 2);
- verify_that_initial_numdocs_is_dependent_of_mode(3, 3);
+ verify_that_initial_numdocs_is_dependent_of_mode();
+
}
@Test
@@ -134,13 +132,11 @@ public class NodeResourcesTuningTest {
@Test
public void require_that_flush_strategy_tls_size_is_set_based_on_available_disk() {
- assertFlushStrategyTlsSize(2 * GB, 10, 0.05);
- assertFlushStrategyTlsSize(7 * GB, 100, 0.07);
- assertFlushStrategyTlsSize(5 * GB, 100, 0.05);
- assertFlushStrategyTlsSize(35 * GB, 500, 0.07);
- assertFlushStrategyTlsSize(84 * GB, 1200, 0.07);
- assertFlushStrategyTlsSize(100 * GB, 1720, 0.07);
- assertFlushStrategyTlsSize(100 * GB, 24000, 0.07);
+ assertFlushStrategyTlsSize(2 * GB, 10);
+ assertFlushStrategyTlsSize(2 * GB, 100);
+ assertFlushStrategyTlsSize(10 * GB, 500);
+ assertFlushStrategyTlsSize(24 * GB, 1200);
+ assertFlushStrategyTlsSize(100 * GB, 24000);
}
@Test
@@ -181,8 +177,8 @@ public class NodeResourcesTuningTest {
assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().each().maxmemory());
}
- private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb, double tlsSizeFraction) {
- assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb, tlsSizeFraction).flush().memory().maxtlssize());
+ private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) {
+ assertEquals(expTlsSizeBytes, configFromDiskSetting(diskGb).flush().memory().maxtlssize());
}
private static void assertSummaryReadIo(ProtonConfig.Summary.Read.Io.Enum expValue, boolean fastDisk) {
@@ -197,23 +193,16 @@ public class NodeResourcesTuningTest {
assertEquals(sharedDisk, configFromEnvironmentType(docker).hwinfo().disk().shared());
}
- private static void assertWriteFilter(double expMemoryLimit, int memoryGb) {
- assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, 0).writefilter().memorylimit(), delta);
- }
-
private static ProtonConfig configFromDiskSetting(boolean fastDisk) {
return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk));
}
private static ProtonConfig configFromDiskSetting(int diskGb) {
- return configFromDiskSetting(diskGb, 0.07);
- }
- private static ProtonConfig configFromDiskSetting(int diskGb, double tlsSizeFraction) {
- return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0, tlsSizeFraction);
+ return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0);
}
private static ProtonConfig configFromMemorySetting(double memoryGb, double fractionOfMemoryReserved) {
- return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved, 0.07);
+ return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved);
}
private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) {
@@ -239,26 +228,26 @@ public class NodeResourcesTuningTest {
return getConfig(flavorBuilder, new ProtonConfig.Builder());
}
- private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) {
- return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved, tlsSizeFraction);
+ private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved) {
+ return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved);
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) {
return getConfig(flavorBuilder, protonBuilder,1);
}
- private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved, double tlsSizeFraction) {
- return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved, tlsSizeFraction);
+ private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) {
+ return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved);
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
int numThreadsPerSearch) {
- return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, 0.07);
+ return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0);
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
- int numThreadsPerSearch, double fractionOfMemoryReserved, double tlsSizeFraction) {
+ int numThreadsPerSearch, double fractionOfMemoryReserved) {
flavorBuilder.name("my_flavor");
- NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved, tlsSizeFraction);
+ NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java
index d000f83d6cd..226045f4d8a 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java
@@ -51,8 +51,7 @@ public class SchemaNodeTest {
private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey,
NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) {
return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown,
- Optional.empty(), Optional.empty(), isHosted, 0.0,
- root.getDeployState().featureFlags().tlsSizeFraction());
+ Optional.empty(), Optional.empty(), isHosted, 0.0);
}
private static SearchNode createSearchNode(MockRoot root) {