aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJon Bratseth <bratseth@oath.com>2021-12-18 22:48:27 +0100
committerGitHub <noreply@github.com>2021-12-18 22:48:27 +0100
commit63da9a412b31fb1badc787e4ae8550a0c2b50fc0 (patch)
tree5a28371c3f06e0ffe706af7d2dbdb82f663ddaa4
parent295762905a82caf53a375235617a0d612f2d6601 (diff)
parent914735bfed125ac2e85b71214051f367f32e79ea (diff)
Merge pull request #20579 from vespa-engine/balder/expose-memory-percentage-on-the-container-cluster
Balder/expose memory percentage on the container cluster
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java20
-rwxr-xr-xconfig-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java5
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java18
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java15
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java14
-rw-r--r--config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java26
-rw-r--r--config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java37
-rwxr-xr-xconfig-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java25
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java49
-rw-r--r--config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java13
10 files changed, 129 insertions, 93 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
index e8be43fdc96..c4d420f2d44 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java
@@ -167,11 +167,17 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
public void setMemoryPercentage(Integer memoryPercentage) { this.memoryPercentage = memoryPercentage;
}
- /**
- * Returns the percentage of host physical memory this application has specified for nodes in this cluster,
- * or empty if this is not specified by the application.
- */
- public Optional<Integer> getMemoryPercentage() { return Optional.ofNullable(memoryPercentage); }
+ @Override
+ public Optional<Integer> getMemoryPercentage() {
+ if (memoryPercentage != null) {
+ return Optional.of(memoryPercentage);
+ } else if (isHostedVespa()) {
+ return getHostClusterId().isPresent() ?
+ Optional.of(heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster) :
+ Optional.of(heapSizePercentageOfTotalNodeMemory);
+ }
+ return Optional.empty();
+ }
/*
Create list of endpoints, these will be consumed later by the LBservicesProducer
@@ -291,10 +297,6 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat
.heapsize(1536);
if (getMemoryPercentage().isPresent()) {
builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get());
- } else if (isHostedVespa()) {
- builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ?
- heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster :
- heapSizePercentageOfTotalNodeMemory);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
index a9ce03d1d55..c73a3b2a676 100755
--- a/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ContainerCluster.java
@@ -653,4 +653,9 @@ public abstract class ContainerCluster<CONTAINER extends Container>
public boolean getDeferChangesUntilRestart() { return deferChangesUntilRestart; }
+ /**
+ * Returns the percentage of host physical memory this application has specified for nodes in this cluster,
+ * or empty if this is not specified by the application.
+ */
+ public Optional<Integer> getMemoryPercentage() { return Optional.empty(); }
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
index f981ed6228f..fb6e2dc0bd8 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java
@@ -74,21 +74,21 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
private final int defaultMaxCompactBuffers;
/** Whether the nodes of this cluster also hosts a container cluster in a hosted system */
- private final boolean combined;
+ private final double fractionOfMemoryReserved;
public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> {
private final Map<String, NewDocumentType> documentDefinitions;
private final Set<NewDocumentType> globallyDistributedDocuments;
- private final boolean combined;
+ private final double fractionOfMemoryReserved;
private final ResourceLimits resourceLimits;
public Builder(Map<String, NewDocumentType> documentDefinitions,
Set<NewDocumentType> globallyDistributedDocuments,
- boolean combined, ResourceLimits resourceLimits) {
+ double fractionOfMemoryReserved, ResourceLimits resourceLimits) {
this.documentDefinitions = documentDefinitions;
this.globallyDistributedDocuments = globallyDistributedDocuments;
- this.combined = combined;
+ this.fractionOfMemoryReserved = fractionOfMemoryReserved;
this.resourceLimits = resourceLimits;
}
@@ -106,7 +106,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
globallyDistributedDocuments,
getFlushOnShutdown(flushOnShutdownElem),
syncTransactionLog,
- combined);
+ fractionOfMemoryReserved);
ModelElement tuning = clusterElem.childByPath("engine.proton.tuning");
if (tuning != null) {
@@ -208,7 +208,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
Set<NewDocumentType> globallyDistributedDocuments,
boolean flushOnShutdown,
Boolean syncTransactionLog,
- boolean combined)
+ double fractionOfMemoryReserved)
{
super(parent, "search");
this.clusterName = clusterName;
@@ -217,7 +217,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
this.flushOnShutdown = flushOnShutdown;
this.syncTransactionLog = syncTransactionLog;
- this.combined = combined;
+ this.fractionOfMemoryReserved = fractionOfMemoryReserved;
this.feedSequencerType = convertFeedSequencerType(featureFlags.feedSequencerType());
this.feedTaskLimit = featureFlags.feedTaskLimit();
this.feedMasterTaskLimit = featureFlags.feedMasterTaskLimit();
@@ -290,7 +290,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
Optional<Tuning> tuning = Optional.ofNullable(this.tuning);
if (element == null) {
searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec,
- clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), combined);
+ clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), fractionOfMemoryReserved);
searchNode.setHostResource(node.getHostResource());
searchNode.initService(deployState.getDeployLogger());
@@ -298,7 +298,7 @@ public class ContentSearchCluster extends AbstractConfigProducer<SearchCluster>
tls.setHostResource(searchNode.getHostResource());
tls.initService(deployState.getDeployLogger());
} else {
- searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, combined).build(deployState, parent, element.getXml());
+ searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, fractionOfMemoryReserved).build(deployState, parent, element.getXml());
tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml());
}
searchNode.setTls(tls);
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
index 0c0b2b80d79..9ab684c3f9d 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java
@@ -131,7 +131,7 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce
resourceLimits.getClusterControllerLimits()).build(deployState, c, contentElement.getXml());
c.search = new ContentSearchCluster.Builder(documentDefinitions,
globallyDistributedDocuments,
- isCombined(getClusterId(contentElement), containers),
+ fractionOfMemoryReserved(getClusterId(contentElement), containers),
resourceLimits.getContentNodeLimits()).build(deployState, c, contentElement.getXml());
c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c);
c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement);
@@ -241,11 +241,14 @@ public class ContentCluster extends AbstractConfigProducer<AbstractConfigProduce
}
/** Returns whether this hosts one of the given container clusters */
- private boolean isCombined(String clusterId, Collection<ContainerModel> containers) {
- return containers.stream()
- .map(model -> model.getCluster().getHostClusterId())
- .filter(Optional::isPresent)
- .anyMatch(id -> id.get().equals(clusterId));
+ private double fractionOfMemoryReserved(String clusterId, Collection<ContainerModel> containers) {
+ for (ContainerModel containerModel : containers) {
+ Optional<String> hostClusterId = containerModel.getCluster().getHostClusterId();
+ if (hostClusterId.isPresent() && hostClusterId.get().equals(clusterId) && containerModel.getCluster().getMemoryPercentage().isPresent()) {
+ return containerModel.getCluster().getMemoryPercentage().get() * 0.01;
+ }
+ }
+ return 0.0;
}
private void setupExperimental(ContentCluster cluster, ModelElement experimental) {
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
index 65efcf85b1d..652029dfef3 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java
@@ -3,7 +3,6 @@ package com.yahoo.vespa.model.search;
import com.yahoo.config.provision.NodeResources;
import com.yahoo.vespa.config.search.core.ProtonConfig;
-import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import static java.lang.Long.min;
@@ -20,17 +19,17 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
private final static long MEMORY_COST_PER_DOCUMENT_STORE_ONLY = 46L;
private final NodeResources resources;
private final int threadsPerSearch;
- private final boolean combined;
+ private final double fractionOfMemoryReserved;
// "Reserve" 0.5GB of memory for other processes running on the content node (config-proxy, metrics-proxy).
public static final double reservedMemoryGb = 0.5;
public NodeResourcesTuning(NodeResources resources,
int threadsPerSearch,
- boolean combined) {
+ double fractionOfMemoryReserved) {
this.resources = resources;
this.threadsPerSearch = threadsPerSearch;
- this.combined = combined;
+ this.fractionOfMemoryReserved = fractionOfMemoryReserved;
}
@Override
@@ -122,12 +121,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer {
/** Returns the memory we can expect will be available for the content node processes */
private double usableMemoryGb() {
double usableMemoryGb = resources.memoryGb() - reservedMemoryGb;
- if ( ! combined) {
- return usableMemoryGb;
- }
-
- double fractionTakenByContainer = ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster * 1e-2;
- return usableMemoryGb * (1 - fractionTakenByContainer);
+ return usableMemoryGb * (1 - fractionOfMemoryReserved);
}
}
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
index 4b3b464f9e5..6a4be692b0c 100644
--- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
+++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java
@@ -67,9 +67,7 @@ public class SearchNode extends AbstractService implements
private AbstractService serviceLayerService;
private final Optional<Tuning> tuning;
private final Optional<ResourceLimits> resourceLimits;
-
- /** Whether this search node is co-located with a container node on a hosted system */
- private final boolean combined;
+ private final double fractionOfMemoryReserved;
public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> {
@@ -80,11 +78,11 @@ public class SearchNode extends AbstractService implements
private final boolean flushOnShutdown;
private final Optional<Tuning> tuning;
private final Optional<ResourceLimits> resourceLimits;
- private boolean combined;
+ private final double fractionOfMemoryReserved;
public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node,
boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits,
- boolean combined) {
+ double fractionOfMemoryReserved) {
this.name = name;
this.nodeSpec = nodeSpec;
this.clusterName = clusterName;
@@ -92,13 +90,13 @@ public class SearchNode extends AbstractService implements
this.flushOnShutdown = flushOnShutdown;
this.tuning = tuning;
this.resourceLimits = resourceLimits;
- this.combined = combined;
+ this.fractionOfMemoryReserved = fractionOfMemoryReserved;
}
@Override
protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) {
return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode,
- flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), combined);
+ flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), fractionOfMemoryReserved);
}
}
@@ -106,16 +104,16 @@ public class SearchNode extends AbstractService implements
public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec,
String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown,
Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
- boolean combined) {
+ double fractionOfMemoryReserved) {
return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService,
- flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined);
+ flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved);
}
private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec,
String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown,
Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
- boolean combined) {
- this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined);
+ double fractionOfMemoryReserved) {
+ this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved);
this.distributionKey = distributionKey;
this.serviceLayerService = serviceLayerService;
setPropertiesElastic(clusterName, distributionKey);
@@ -123,11 +121,11 @@ public class SearchNode extends AbstractService implements
private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName,
boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa,
- boolean combined) {
+ double fractionOfMemoryReserved) {
super(parent, name);
setOmpNumThreads(1);
this.isHostedVespa = isHostedVespa;
- this.combined = combined;
+ this.fractionOfMemoryReserved = fractionOfMemoryReserved;
this.nodeSpec = nodeSpec;
this.clusterName = clusterName;
this.flushOnShutdown = flushOnShutdown;
@@ -281,7 +279,7 @@ public class SearchNode extends AbstractService implements
if (nodeResources.isPresent()) {
var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(),
tuning.map(Tuning::threadsPerSearch).orElse(1),
- combined);
+ fractionOfMemoryReserved);
nodeResourcesTuning.getConfig(builder);
tuning.ifPresent(t -> t.getConfig(builder));
diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
index 171958f2bfa..912161019fe 100644
--- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
+++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java
@@ -278,6 +278,43 @@ public class ModelProvisioningTest {
}
}
+ @Test
+ public void testCombinedClusterWithJvmHeapSizeOverride() {
+ var containerElements = Set.of("jdisc", "container");
+ for (var containerElement : containerElements) {
+ String xmlWithNodes =
+ "<?xml version='1.0' encoding='utf-8' ?>" +
+ "<services>" +
+ " <" + containerElement + " version='1.0' id='container1'>" +
+ " <search/>" +
+ " <nodes of='content1'>" +
+ " <jvm allocated-memory=\"30%\"/>" +
+ " </nodes>" +
+ " </" + containerElement + ">" +
+ " <content version='1.0' id='content1'>" +
+ " <redundancy>2</redundancy>" +
+ " <documents>" +
+ " <document type='type1' mode='index'/>" +
+ " </documents>" +
+ " <nodes count='2'>" +
+ " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" +
+ " </nodes>" +
+ " </content>" +
+ "</services>";
+ VespaModelTester tester = new VespaModelTester();
+ tester.addHosts(5);
+ VespaModel model = tester.createModel(xmlWithNodes, true);
+ assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size());
+ assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size());
+ assertEquals("Heap size is lowered with combined clusters",
+ 30, physicalMemoryPercentage(model.getContainerClusters().get("container1")));
+ assertEquals("Memory for proton is lowered to account for the jvm heap",
+ (long)((3 - reservedMemoryGb) * (Math.pow(1024, 3)) * (1 - 0.30)), protonMemorySize(model.getContentClusters().get("content1")));
+ assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model);
+ assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model);
+ }
+ }
+
/** For comparison with the above */
@Test
public void testNonCombinedCluster() {
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
index 560ac28b6f7..bcd29267277 100755
--- a/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/container/ContainerClusterTest.java
@@ -40,7 +40,6 @@ import java.util.Iterator;
import java.util.List;
import java.util.Objects;
import java.util.OptionalInt;
-import java.util.OptionalLong;
import java.util.Set;
import java.util.stream.Collectors;
@@ -51,9 +50,7 @@ import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.applic
import static com.yahoo.config.model.api.ApplicationClusterEndpoint.Scope.global;
import static com.yahoo.config.provision.SystemName.cd;
import static com.yahoo.config.provision.SystemName.main;
-import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.hasItem;
-import static org.hamcrest.Matchers.hasKey;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertThat;
@@ -100,8 +97,8 @@ public class ContainerClusterTest {
assertEquals("cd", config.system());
}
- private ApplicationContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster) {
- return createContainerCluster(root, isCombinedCluster, null);
+ private ApplicationContainerCluster createContainerCluster(MockRoot root) {
+ return createContainerCluster(root, false, null);
}
private ApplicationContainerCluster createContainerCluster(MockRoot root, boolean isCombinedCluster, Integer memoryPercentage) {
ApplicationContainerCluster cluster = new ApplicationContainerCluster(root, "container0", "container1", root.getDeployState());
@@ -162,7 +159,7 @@ public class ContainerClusterTest {
private void verifyJvmArgs(boolean isHosted, boolean hasDocProc) {
MockRoot root = createRoot(isHosted);
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
if (hasDocProc) {
cluster.setDocproc(new ContainerDocproc(cluster, null));
}
@@ -227,7 +224,7 @@ public class ContainerClusterTest {
public void requireThatJvmOmitStackTraceInFastThrowOptionWorks() {
// Empty option if option not set in property
MockRoot root = createRoot(new DeployState.Builder().build());
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
ApplicationContainer container = cluster.getContainers().get(0);
assertEquals("", container.getJvmOptions());
@@ -235,7 +232,7 @@ public class ContainerClusterTest {
String jvmOption = "-XX:-foo";
DeployState deployState = new DeployState.Builder().properties(new TestProperties().setJvmOmitStackTraceInFastThrowOption(jvmOption)).build();
root = createRoot(deployState);
- cluster = createContainerCluster(root, false);
+ cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
container = cluster.getContainers().get(0);
assertEquals(jvmOption, container.getJvmOptions());
@@ -244,7 +241,7 @@ public class ContainerClusterTest {
@Test
public void requireThatWeCanHandleNull() {
MockRoot root = createRoot(false);
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
Container container = cluster.getContainers().get(0);
container.setJvmOptions("");
@@ -256,7 +253,7 @@ public class ContainerClusterTest {
@Test
public void requireThatNonHostedUsesExpectedDefaultThreadpoolConfiguration() {
MockRoot root = new MockRoot("foo");
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
root.freezeModelTopology();
@@ -268,13 +265,13 @@ public class ContainerClusterTest {
@Test
public void container_cluster_has_default_threadpool_provider() {
MockRoot root = new MockRoot("foo");
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
root.freezeModelTopology();
ComponentId expectedComponentId = new ComponentId("default-threadpool");
var components = cluster.getComponentsMap();
- assertThat(components, hasKey(expectedComponentId));
+ assertTrue(components.containsKey(expectedComponentId));
Component<?, ?> component = components.get(expectedComponentId);
assertEquals(ThreadPoolProvider.class.getName(), component.getClassId().getName());
}
@@ -287,7 +284,7 @@ public class ContainerClusterTest {
.properties(new TestProperties().setHostedVespa(true))
.applicationPackage(new MockApplicationPackage.Builder().build())
.build());
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
root.freezeModelTopology();
@@ -304,7 +301,7 @@ public class ContainerClusterTest {
.properties(new TestProperties().setHostedVespa(true))
.applicationPackage(new MockApplicationPackage.Builder().build())
.build());
- ApplicationContainerCluster cluster = createContainerCluster(root, false);
+ ApplicationContainerCluster cluster = createContainerCluster(root);
addContainer(root, cluster, "c1", "host-c1");
root.freezeModelTopology();
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
index 649a445e7a5..781266c8223 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java
@@ -5,6 +5,7 @@ import com.yahoo.collections.Pair;
import com.yahoo.config.provision.Flavor;
import com.yahoo.config.provisioning.FlavorsConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
+import com.yahoo.vespa.model.container.ApplicationContainerCluster;
import org.junit.Test;
import java.util.Arrays;
@@ -31,8 +32,8 @@ public class NodeResourcesTuningTest {
@Test
public void require_that_hwinfo_memory_size_is_set() {
- assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, false).hwinfo().memory().size());
- assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, true).hwinfo().memory().size(), 1000);
+ assertEquals(24 * GB, configFromMemorySetting(24 + reservedMemoryGb, 0).hwinfo().memory().size());
+ assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).hwinfo().memory().size(), 1000);
}
@Test
@@ -153,14 +154,14 @@ public class NodeResourcesTuningTest {
@Test
public void require_that_summary_cache_max_bytes_is_set_based_on_memory() {
- assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, false).summary().cache().maxbytes());
- assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, false).summary().cache().maxbytes());
+ assertEquals(1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, 0).summary().cache().maxbytes());
+ assertEquals(256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, 0).summary().cache().maxbytes());
}
@Test
public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() {
- assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000);
- assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, true).summary().cache().maxbytes(), 1000);
+ assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000);
+ assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256 + reservedMemoryGb, ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster*0.01).summary().cache().maxbytes(), 1000);
}
@Test
@@ -169,12 +170,12 @@ public class NodeResourcesTuningTest {
}
private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int wantedMemoryGb) {
- assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).summary().log().maxfilesize());
+ assertEquals(expFileSizeBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).summary().log().maxfilesize());
}
private static void assertFlushStrategyMemory(long expMemoryBytes, int wantedMemoryGb) {
- assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().maxmemory());
- assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, false).flush().memory().each().maxmemory());
+ assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().maxmemory());
+ assertEquals(expMemoryBytes, configFromMemorySetting(wantedMemoryGb + reservedMemoryGb, 0).flush().memory().each().maxmemory());
}
private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) {
@@ -194,55 +195,55 @@ public class NodeResourcesTuningTest {
}
private static void assertWriteFilter(double expMemoryLimit, int memoryGb) {
- assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, false).writefilter().memorylimit(), delta);
+ assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, 0).writefilter().memorylimit(), delta);
}
private static ProtonConfig configFromDiskSetting(boolean fastDisk) {
- return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk), false);
+ return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk), 0);
}
private static ProtonConfig configFromDiskSetting(int diskGb) {
- return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), false);
+ return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), 0);
}
- private static ProtonConfig configFromMemorySetting(double memoryGb, boolean combined) {
- return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), combined);
+ private static ProtonConfig configFromMemorySetting(double memoryGb, double fractionOfMemoryReserved) {
+ return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved);
}
private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) {
return getConfig(new FlavorsConfig.Flavor.Builder()
- .minMainMemoryAvailableGb(memoryGb), builder, false);
+ .minMainMemoryAvailableGb(memoryGb), builder, 0);
}
private static ProtonConfig configFromNumCoresSetting(double numCores) {
- return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), false);
+ return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), 0);
}
private static ProtonConfig configFromNumCoresSetting(double numCores, int numThreadsPerSearch) {
return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores),
- new ProtonConfig.Builder(), numThreadsPerSearch, false);
+ new ProtonConfig.Builder(), numThreadsPerSearch, 0);
}
private static ProtonConfig configFromEnvironmentType(boolean docker) {
String environment = (docker ? "DOCKER_CONTAINER" : "undefined");
- return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment), false);
+ return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment), 0);
}
- private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, boolean combined) {
- return getConfig(flavorBuilder, new ProtonConfig.Builder(), combined);
+ private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, double fractionOfMemoryReserved) {
+ return getConfig(flavorBuilder, new ProtonConfig.Builder(), fractionOfMemoryReserved);
}
- private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, boolean combined) {
+ private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) {
flavorBuilder.name("my_flavor");
- NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), 1, combined);
+ NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), 1, fractionOfMemoryReserved);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}
private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder,
- int numThreadsPerSearch, boolean combined) {
+ int numThreadsPerSearch, double fractionOfMemoryReserved) {
flavorBuilder.name("my_flavor");
- NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, combined);
+ NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved);
tuning.getConfig(protonBuilder);
return new ProtonConfig(protonBuilder);
}
diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java
index cca2d817df4..e8781904786 100644
--- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java
+++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SchemaNodeTest.java
@@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search.test;
import com.yahoo.config.model.api.ModelContext;
import com.yahoo.config.model.deploy.DeployState;
import com.yahoo.config.model.deploy.TestProperties;
-import com.yahoo.config.model.producer.AbstractConfigProducer;
import com.yahoo.config.model.test.MockRoot;
import com.yahoo.searchlib.TranslogserverConfig;
import com.yahoo.vespa.config.search.core.ProtonConfig;
@@ -52,12 +51,12 @@ public class SchemaNodeTest {
}
private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey,
- NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, boolean combined) {
- return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined);
+ NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) {
+ return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, 0.0);
}
private static SearchNode createSearchNode(MockRoot root) {
- return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true, false);
+ return createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), true, true);
}
@Test
@@ -70,7 +69,7 @@ public class SchemaNodeTest {
@Test
public void requireThatBasedirIsCorrectForElasticMode() {
MockRoot root = new MockRoot("");
- SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false);
+ SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted());
prepare(root, node, true);
assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node);
}
@@ -78,7 +77,7 @@ public class SchemaNodeTest {
@Test
public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() {
MockRoot root = new MockRoot("");
- SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false);
+ SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted());
node.setHostResource(new HostResource(new Host(node, "mynbode")));
node.initService(root.deployLogger());
assertFalse(node.getPreShutdownCommand().isPresent());
@@ -87,7 +86,7 @@ public class SchemaNodeTest {
@Test
public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() {
MockRoot root = new MockRoot("");
- SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), false);
+ SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted());
node.setHostResource(new HostResource(new Host(node, "mynbode2")));
node.initService(root.deployLogger());
assertTrue(node.getPreShutdownCommand().isPresent());