diff options
author | Jon Bratseth <bratseth@oath.com> | 2020-06-02 11:20:33 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-06-02 11:20:33 +0200 |
commit | c33cef434cfc50f2192eab97cffc1a07731399d9 (patch) | |
tree | 05cc5d06081d3f2b7bd40024b0a104ae90f9c421 | |
parent | b04134020615e37e508f9e92e9f4bdad82bd709f (diff) | |
parent | 0e4619ad522882965a1494808d1aefc0fd696d98 (diff) |
Merge pull request #13441 from vespa-engine/bratseth/account-for-combined-memory
Bratseth/account for combined memory
18 files changed, 224 insertions, 141 deletions
diff --git a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java index 761c3645bad..b9e7976ccf7 100644 --- a/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java +++ b/config-model/src/main/java/com/yahoo/config/model/ConfigModel.java @@ -40,7 +40,7 @@ public abstract class ConfigModel { * * @param configModelRepo The ConfigModelRepo of the VespaModel */ - public void initialize(ConfigModelRepo configModelRepo) { return; } + public void initialize(ConfigModelRepo configModelRepo) { } /** * Prepares this model to start serving config requests, possibly using properties of other models. diff --git a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java index 56e0bd579cb..bb5ba71700c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/container/ApplicationContainerCluster.java @@ -58,6 +58,10 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat public static final String METRICS_V2_HANDLER_BINDING_1 = "http://*" + MetricsV2Handler.V2_PATH; public static final String METRICS_V2_HANDLER_BINDING_2 = METRICS_V2_HANDLER_BINDING_1 + "/*"; + public static final int heapSizePercentageOfTotalNodeMemory = 60; + public static final int heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster = 17; + + private final Set<FileReference> applicationBundles = new LinkedHashSet<>(); private final ConfigProducerGroup<Servlet> servletGroup; @@ -218,13 +222,15 @@ public final class ApplicationContainerCluster extends ContainerCluster<Applicat super.getConfig(builder); builder.jvm.verbosegc(true) .availableProcessors(0) - .compressedClassSpaceSize(0) //TODO Reduce, next step is 512m + .compressedClassSpaceSize(0) .minHeapsize(1536) .heapsize(1536); if (getMemoryPercentage().isPresent()) { builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getMemoryPercentage().get()); } else if (isHostedVespa()) { - builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? 17 : 60); + builder.jvm.heapSizeAsPercentageOfPhysicalMemory(getHostClusterId().isPresent() ? + heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster : + heapSizePercentageOfTotalNodeMemory); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java index b3f2b81014b..404a0ceb4ea 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/Content.java @@ -39,7 +39,6 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Optional; import java.util.Set; -import java.util.logging.Logger; /** * The config model from a content tag in services. @@ -49,8 +48,6 @@ import java.util.logging.Logger; */ public class Content extends ConfigModel { - private static final Logger log = Logger.getLogger(Content.class.getName()); - private ContentCluster cluster; private Optional<ApplicationContainerCluster> ownedIndexingCluster = Optional.empty(); @@ -207,6 +204,11 @@ public class Content extends ConfigModel { content.cluster = new ContentCluster.Builder(admin).build(content.containers, modelContext, xml); buildIndexingClusters(content, modelContext, (ApplicationConfigProducerRoot)modelContext.getParentProducer()); + markCombinedClusters(); + } + + private void markCombinedClusters() { + } /** Select/creates and initializes the indexing cluster coupled to this */ diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 1bf9bc10be4..6cb45c4559b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -60,32 +60,39 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot private Map<StorageGroup, NodeSpec> groupToSpecMap = new LinkedHashMap<>(); private Optional<ResourceLimits> resourceLimits = Optional.empty(); + /** Whether the nodes of this cluster also hosts a container cluster in a hosted system */ + private final boolean combined; + public void prepare() { - List<SearchNode> allBackends = getSearchNodes(); - for (AbstractSearchCluster cluster : clusters.values()) { - cluster.prepareToDistributeFiles(allBackends); - } + clusters.values().forEach(cluster -> cluster.prepareToDistributeFiles(getSearchNodes())); } public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<ContentSearchCluster> { private final Map<String, NewDocumentType> documentDefinitions; private final Set<NewDocumentType> globallyDistributedDocuments; + private final boolean combined; public Builder(Map<String, NewDocumentType> documentDefinitions, - Set<NewDocumentType> globallyDistributedDocuments) { + Set<NewDocumentType> globallyDistributedDocuments, + boolean combined) { this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; + this.combined = combined; } @Override protected ContentSearchCluster doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { ModelElement clusterElem = new ModelElement(producerSpec); - String clusterName = ContentCluster.getClusterName(clusterElem); + String clusterName = ContentCluster.getClusterId(clusterElem); Boolean flushOnShutdownElem = clusterElem.childAsBoolean("engine.proton.flush-on-shutdown"); - ContentSearchCluster search = new ContentSearchCluster(ancestor, clusterName, documentDefinitions, globallyDistributedDocuments, - getFlushOnShutdown(flushOnShutdownElem, deployState)); + ContentSearchCluster search = new ContentSearchCluster(ancestor, + clusterName, + documentDefinitions, + globallyDistributedDocuments, + getFlushOnShutdown(flushOnShutdownElem, deployState), + combined); ModelElement tuning = clusterElem.childByPath("engine.proton.tuning"); if (tuning != null) { @@ -171,13 +178,15 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot String clusterName, Map<String, NewDocumentType> documentDefinitions, Set<NewDocumentType> globallyDistributedDocuments, - boolean flushOnShutdown) + boolean flushOnShutdown, + boolean combined) { super(parent, "search"); this.clusterName = clusterName; this.documentDefinitions = documentDefinitions; this.globallyDistributedDocuments = globallyDistributedDocuments; this.flushOnShutdown = flushOnShutdown; + this.combined = combined; } public void setVisibilityDelay(double delay) { @@ -236,27 +245,27 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot AbstractConfigProducer parent = hasIndexedCluster() ? getIndexed() : this; NodeSpec spec = getNextSearchNodeSpec(parentGroup); - SearchNode snode; + SearchNode searchNode; TransactionLogServer tls; Optional<Tuning> tuning = Optional.ofNullable(this.tuning); if (element == null) { - snode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, - clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted()); - snode.setHostResource(node.getHostResource()); - snode.initService(deployState.getDeployLogger()); + searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, + clusterName, node, flushOnShutdown, tuning, resourceLimits, parentGroup.isHosted(), combined); + searchNode.setHostResource(node.getHostResource()); + searchNode.initService(deployState.getDeployLogger()); - tls = new TransactionLogServer(snode, clusterName); - tls.setHostResource(snode.getHostResource()); + tls = new TransactionLogServer(searchNode, clusterName); + tls.setHostResource(searchNode.getHostResource()); tls.initService(deployState.getDeployLogger()); } else { - snode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits).build(deployState, parent, element.getXml()); - tls = new TransactionLogServer.Builder(clusterName).build(deployState, snode, element.getXml()); + searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, combined).build(deployState, parent, element.getXml()); + tls = new TransactionLogServer.Builder(clusterName).build(deployState, searchNode, element.getXml()); } - snode.setTls(tls); + searchNode.setTls(tls); if (hasIndexedCluster()) { - getIndexed().addSearcher(snode); + getIndexed().addSearcher(searchNode); } else { - nonIndexed.add(snode); + nonIndexed.add(searchNode); } } @@ -359,9 +368,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot int numDocumentDbs = builder.documentdb.size(); builder.initialize(new ProtonConfig.Initialize.Builder().threads(numDocumentDbs + 1)); - if (resourceLimits.isPresent()) { - resourceLimits.get().getConfig(builder); - } + resourceLimits.ifPresent(limits -> limits.getConfig(builder)); if (tuning != null) { tuning.getConfig(builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java index 6dd3e619ec2..9fb553fffde 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/cluster/ContentCluster.java @@ -60,6 +60,7 @@ import java.util.Comparator; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.TreeMap; import java.util.logging.Level; @@ -125,11 +126,14 @@ public class ContentCluster extends AbstractConfigProducer implements RedundancyBuilder redundancyBuilder = new RedundancyBuilder(contentElement); Set<NewDocumentType> globallyDistributedDocuments = new GlobalDistributionBuilder(documentDefinitions).build(documentsElement); - ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterName(contentElement), documentDefinitions, + ContentCluster c = new ContentCluster(context.getParentProducer(), getClusterId(contentElement), documentDefinitions, globallyDistributedDocuments, routingSelection, deployState.zone(), deployState.isHosted()); - c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterName(contentElement), contentElement).build(deployState, c, contentElement.getXml()); - c.search = new ContentSearchCluster.Builder(documentDefinitions, globallyDistributedDocuments).build(deployState, c, contentElement.getXml()); + c.clusterControllerConfig = new ClusterControllerConfig.Builder(getClusterId(contentElement), contentElement).build(deployState, c, contentElement.getXml()); + c.search = new ContentSearchCluster.Builder(documentDefinitions, + globallyDistributedDocuments, + isCombined(getClusterId(contentElement), containers)) + .build(deployState, c, contentElement.getXml()); c.persistenceFactory = new EngineFactoryBuilder().build(contentElement, c); c.storageNodes = new StorageCluster.Builder().build(deployState, c, w3cContentElement); c.distributorNodes = new DistributorCluster.Builder(c).build(deployState, c, w3cContentElement); @@ -237,6 +241,14 @@ public class ContentCluster extends AbstractConfigProducer implements } } + /** Returns whether this hosts one of the given container clusters */ + private boolean isCombined(String clusterId, Collection<ContainerModel> containers) { + return containers.stream() + .map(model -> model.getCluster().getHostClusterId()) + .filter(Optional::isPresent) + .anyMatch(id -> id.get().equals(clusterId)); + } + private void setupExperimental(ContentCluster cluster, ModelElement experimental) { // Put handling of experimental flags here } @@ -496,9 +508,8 @@ public class ContentCluster extends AbstractConfigProducer implements public void prepare(DeployState deployState) { search.prepare(); - if (clusterControllers != null) { + if (clusterControllers != null) clusterControllers.prepare(deployState); - } } /** Returns cluster controllers if this is multitenant, null otherwise */ @@ -509,13 +520,9 @@ public class ContentCluster extends AbstractConfigProducer implements return getPersistence().getDefaultDistributionMode(); } - public static String getClusterName(ModelElement clusterElem) { - String clusterName = clusterElem.stringAttribute("id"); - if (clusterName == null) { - clusterName = "content"; - } - - return clusterName; + public static String getClusterId(ModelElement clusterElem) { + String clusterId = clusterElem.stringAttribute("id"); + return clusterId != null ? clusterId : "content"; } public String getName() { return clusterName; } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java index 51fc610bf28..36e4554e610 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorServerProducer.java @@ -14,15 +14,15 @@ public class StorServerProducer implements StorServerConfig.Producer { ModelElement tuning = element.child("tuning"); if (tuning == null) { - return new StorServerProducer(ContentCluster.getClusterName(element), null, null); + return new StorServerProducer(ContentCluster.getClusterId(element), null, null); } ModelElement merges = tuning.child("merges"); if (merges == null) { - return new StorServerProducer(ContentCluster.getClusterName(element), null, null); + return new StorServerProducer(ContentCluster.getClusterId(element), null, null); } - return new StorServerProducer(ContentCluster.getClusterName(element), + return new StorServerProducer(ContentCluster.getClusterId(element), merges.integerAttribute("max-per-node"), merges.integerAttribute("max-queue-size")); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java index 091e82a9c76..2e5594a001d 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/storagecluster/StorageCluster.java @@ -35,7 +35,7 @@ public class StorageCluster extends AbstractConfigProducer<StorageNode> final ContentCluster cluster = (ContentCluster)ancestor; return new StorageCluster(ancestor, - ContentCluster.getClusterName(clusterElem), + ContentCluster.getClusterId(clusterElem), new FileStorProducer.Builder().build(cluster, clusterElem), new IntegrityCheckerProducer.Builder().build(cluster, clusterElem), new StorServerProducer.Builder().build(clusterElem), diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java index fe6c6c52e2d..58d608ec9f9 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/AbstractSearchCluster.java @@ -75,7 +75,6 @@ public abstract class AbstractSearchCluster extends AbstractConfigProducer public abstract int getRowBits(); public final void setClusterIndex(int index) { this.index = index; } public final int getClusterIndex() { return index; } - protected abstract void assureSdConsistent(); @Override public abstract void getConfig(DocumentdbInfoConfig.Builder builder); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 5e5976c4b9c..ef8c236fd94 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -252,9 +252,6 @@ public class IndexedSearchCluster extends SearchCluster unionCfg.getConfig(builder); } - @Override - protected void exportSdFiles(File toDir) { } - boolean useFixedRowInDispatch() { for (SearchNode node : getSearchNodes()) { if (node.getNodeSpec().groupIndex() > 0) { @@ -337,9 +334,6 @@ public class IndexedSearchCluster extends SearchCluster } @Override - protected void assureSdConsistent() { } - - @Override public int getRowBits() { return 8; } /** diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index ee854aaa9c3..c36d3c1dabb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -3,6 +3,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; +import com.yahoo.vespa.model.container.ApplicationContainerCluster; import static java.lang.Long.min; @@ -19,16 +20,18 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final int redundancy; private final int searchableCopies; private final int threadsPerSearch; + private final boolean combined; - - public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies) { - this(resources, redundancy, searchableCopies, 1); - } - public NodeResourcesTuning(NodeResources resources, int redundancy, int searchableCopies, int threadsPerSearch) { + public NodeResourcesTuning(NodeResources resources, + int redundancy, + int searchableCopies, + int threadsPerSearch, + boolean combined) { this.resources = resources; this.redundancy = redundancy; this.searchableCopies = searchableCopies; this.threadsPerSearch = threadsPerSearch; + this.combined = combined; } @Override @@ -51,21 +54,21 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private void getConfig(ProtonConfig.Documentdb.Builder builder) { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)resources.memoryGb() * GB / 64L; + long numDocs = (long)usableMemoryGb() * GB / 64L; builder.allocation.initialnumdocs(numDocs/Math.max(searchableCopies, redundancy)); } } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { - long memoryLimitBytes = (long) ((resources.memoryGb() * 0.05) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() * 0.05) * GB); builder.maxbytes(memoryLimitBytes); } private void setHwInfo(ProtonConfig.Builder builder) { builder.hwinfo.disk.shared(true); builder.hwinfo.cpu.cores((int)resources.vcpu()); - builder.hwinfo.memory.size((long)resources.memoryGb() * GB); - builder.hwinfo.disk.size((long)resources.diskGb() * GB); + builder.hwinfo.memory.size((long)(usableMemoryGb() * GB)); + builder.hwinfo.disk.size((long)(resources.diskGb() * GB)); } private void tuneDiskWriteSpeed(ProtonConfig.Builder builder) { @@ -75,7 +78,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneDocumentStoreMaxFileSize(ProtonConfig.Summary.Log.Builder builder) { - double memoryGb = resources.memoryGb(); + double memoryGb = usableMemoryGb(); long fileSizeBytes = 4 * GB; if (memoryGb <= 12.0) { fileSizeBytes = 256 * MB; @@ -88,7 +91,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { } private void tuneFlushStrategyMemoryLimits(ProtonConfig.Flush.Memory.Builder builder) { - long memoryLimitBytes = (long) ((resources.memoryGb() / 8) * GB); + long memoryLimitBytes = (long) ((usableMemoryGb() / 8) * GB); builder.maxmemory(memoryLimitBytes); builder.each.maxmemory(memoryLimitBytes); } @@ -122,8 +125,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { // "Reserve" 1GB of memory for other processes running on the content node (config-proxy, cluster-controller, metrics-proxy) double reservedMemoryGb = 1; double defaultMemoryLimit = new ProtonConfig.Writefilter(new ProtonConfig.Writefilter.Builder()).memorylimit(); - double scaledMemoryLimit = ((resources.memoryGb() - reservedMemoryGb) * defaultMemoryLimit) / resources.memoryGb(); + double scaledMemoryLimit = ((usableMemoryGb() - reservedMemoryGb) * defaultMemoryLimit) / usableMemoryGb(); builder.memorylimit(scaledMemoryLimit); } + /** Returns the memory we can expect will be available for the content node processes */ + private double usableMemoryGb() { + if ( ! combined ) return resources.memoryGb(); + + double fractionTakenByContainer = (double)ApplicationContainerCluster.heapSizePercentageOfTotalNodeMemoryWhenCombinedCluster / 100; + return resources.memoryGb() * (1 - fractionTakenByContainer); + } + } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java index 0139e949c7a..64dfb3b5597 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchCluster.java @@ -136,10 +136,5 @@ public abstract class SearchCluster extends AbstractSearchCluster public abstract void defaultDocumentsConfig(); public abstract DerivedConfiguration getSdConfig(); - protected abstract void exportSdFiles(File toDir) throws IOException; - protected final void writeSdFiles(File toDir) throws IOException { - assureSdConsistent(); - exportSdFiles(toDir); - } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index c1498936280..738a4cf66d2 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -50,7 +50,12 @@ public class SearchNode extends AbstractService implements MetricsmanagerConfig.Producer, TranslogserverConfig.Producer { - private static final long serialVersionUID = 1L; + private static final int RPC_PORT = 0; + private static final int UNUSED_1 = 1; + private static final int UNUSED_2 = 2; + private static final int UNUSED_3 = 3; + private static final int HEALTH_PORT = 4; + private final boolean isHostedVespa; private final boolean flushOnShutdown; private NodeSpec nodeSpec; @@ -62,11 +67,9 @@ public class SearchNode extends AbstractService implements private AbstractService serviceLayerService; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; - private static final int RPC_PORT = 0; - private static final int UNUSED_1 = 1; - private static final int UNUSED_2 = 2; - private static final int UNUSED_3 = 3; - private static final int HEALTH_PORT = 4; + + /** Whether this search node is co-located with a container node on a hosted system */ + private final boolean combined; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilder<SearchNode> { @@ -77,8 +80,11 @@ public class SearchNode extends AbstractService implements private final boolean flushOnShutdown; private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; + private boolean combined; + public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, - boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits) { + boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, + boolean combined) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -86,38 +92,41 @@ public class SearchNode extends AbstractService implements this.flushOnShutdown = flushOnShutdown; this.tuning = tuning; this.resourceLimits = resourceLimits; + this.combined = combined; } @Override protected SearchNode doBuild(DeployState deployState, AbstractConfigProducer ancestor, Element producerSpec) { return new SearchNode(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, - flushOnShutdown, tuning, resourceLimits, deployState.isHosted()); + flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), combined); } + } - /** - * Creates a SearchNode in elastic mode. - */ public static SearchNode create(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa) { + Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, + boolean combined) { return new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, - flushOnShutdown, tuning, resourceLimits, isHostedVespa); + flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); } private SearchNode(AbstractConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa) { - this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa); + Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, + boolean combined) { + this(parent, name, nodeSpec, clusterName, flushOnShutdown, tuning, resourceLimits, isHostedVespa, combined); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; setPropertiesElastic(clusterName, distributionKey); } private SearchNode(AbstractConfigProducer parent, String name, NodeSpec nodeSpec, String clusterName, - boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa) { + boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, + boolean combined) { super(parent, name); this.isHostedVespa = isHostedVespa; + this.combined = combined; this.nodeSpec = nodeSpec; this.clusterName = clusterName; this.flushOnShutdown = flushOnShutdown; @@ -270,9 +279,11 @@ public class SearchNode extends AbstractService implements builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60); } if (getHostResource() != null && ! getHostResource().realResources().isUnspecified()) { - var nodeResourcesTuning = tuning.isPresent() - ? new NodeResourcesTuning(getHostResource().realResources(), redundancy, searchableCopies, tuning.get().getNumThreadsPerSearch()) - : new NodeResourcesTuning(getHostResource().realResources(), redundancy, searchableCopies); + var nodeResourcesTuning = new NodeResourcesTuning(getHostResource().realResources(), + redundancy, + searchableCopies, + tuning.map(Tuning::threadsPerSearch).orElse(1), + combined); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java index 28e7b3eb37a..aa735d5ae4c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/StreamingSearchCluster.java @@ -84,13 +84,6 @@ public class StreamingSearchCluster extends SearchCluster implements } @Override - protected void assureSdConsistent() { - if (sdConfig == null) { - throw new IllegalStateException("Search cluster '" + getClusterName() + "' does not have any search definitions"); - } - } - - @Override protected void deriveAllSchemas(List<SchemaSpec> local, DeployState deployState) { if (local.size() == 1) { deriveSingleSearchDefinition(local.get(0).getSearchDefinition().getSearch(), deployState); @@ -112,12 +105,7 @@ public class StreamingSearchCluster extends SearchCluster implements public DerivedConfiguration getSdConfig() { return sdConfig; } - @Override - protected void exportSdFiles(File toDir) throws IOException { - if (sdConfig!=null) { - sdConfig.export(toDir.getCanonicalPath()); - } - } + @Override public void defaultDocumentsConfig() { } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 5984aad7f16..d8a2e67bf9a 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -406,7 +406,7 @@ public class Tuning extends AbstractConfigProducer implements ProtonConfig.Produ if (searchNode != null) searchNode.getConfig(builder); } - public int getNumThreadsPerSearch() { + public int threadsPerSearch() { if (searchNode == null) return 1; if (searchNode.threads == null) return 1; if (searchNode.threads.numThreadsPerSearch == null) return 1; diff --git a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java index 7208d8c5fc1..1be58564d1b 100644 --- a/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java +++ b/config-model/src/test/java/com/yahoo/config/model/provision/ModelProvisioningTest.java @@ -252,22 +252,59 @@ public class ModelProvisioningTest { " <documents>" + " <document type='type1' mode='index'/>" + " </documents>" + - " <nodes count='2'/>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + " </content>" + "</services>"; VespaModelTester tester = new VespaModelTester(); tester.addHosts(2); VespaModel model = tester.createModel(xmlWithNodes, true); - assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); assertEquals("Heap size is lowered with combined clusters", 17, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is lowered to account for the jvm heap", + (long)(3 * (Math.pow(1024, 3)) * (1 - 0.17)), protonMemorySize(model.getContentClusters().get("content1"))); assertProvisioned(0, ClusterSpec.Id.from("container1"), ClusterSpec.Type.container, model); assertProvisioned(2, ClusterSpec.Id.from("content1"), ClusterSpec.Id.from("container1"), ClusterSpec.Type.combined, model); } } + /** For comparison with the above */ + @Test + public void testNonCombinedCluster() { + var containerElements = Set.of("jdisc", "container"); + for (var containerElement : containerElements) { + String xmlWithNodes = + "<?xml version='1.0' encoding='utf-8' ?>" + + "<services>" + + " <" + containerElement + " version='1.0' id='container1'>" + + " <search/>" + + " <nodes count='2'/>" + + " </" + containerElement + ">" + + " <content version='1.0' id='content1'>" + + " <redundancy>2</redundancy>" + + " <documents>" + + " <document type='type1' mode='index'/>" + + " </documents>" + + " <nodes count='2'>" + + " <resources vcpu='1' memory='3Gb' disk='9Gb'/>" + + " </nodes>" + + " </content>" + + "</services>"; + VespaModelTester tester = new VespaModelTester(); + tester.addHosts(4); + VespaModel model = tester.createModel(xmlWithNodes, true); + assertEquals("Nodes in content1", 2, model.getContentClusters().get("content1").getRootGroup().getNodes().size()); + assertEquals("Nodes in container1", 2, model.getContainerClusters().get("container1").getContainers().size()); + assertEquals("Heap size is normal", + 60, physicalMemoryPercentage(model.getContainerClusters().get("container1"))); + assertEquals("Memory for proton is normal", + (long)(3 * (Math.pow(1024, 3))), protonMemorySize(model.getContentClusters().get("content1"))); + } + } + @Test public void testCombinedClusterWithJvmOptions() { String xmlWithNodes = @@ -1741,7 +1778,13 @@ public class ModelProvisioningTest { private int physicalMemoryPercentage(ContainerCluster cluster) { QrStartConfig.Builder b = new QrStartConfig.Builder(); cluster.getConfig(b); - return new QrStartConfig(b).jvm().heapSizeAsPercentageOfPhysicalMemory(); + return b.build().jvm().heapSizeAsPercentageOfPhysicalMemory(); + } + + private long protonMemorySize(ContentCluster cluster) { + ProtonConfig.Builder b = new ProtonConfig.Builder(); + cluster.getSearch().getIndexed().getSearchNode(0).getConfig(b); + return b.build().hwinfo().memory().size(); } @Test diff --git a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java index 98bc4210602..b144ea19980 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/container/search/searchchain/MockSearchClusters.java @@ -26,24 +26,27 @@ public class MockSearchClusters { @Override protected AbstractSearchCluster.IndexingMode getIndexingMode() { return streaming ? AbstractSearchCluster.IndexingMode.STREAMING : AbstractSearchCluster.IndexingMode.REALTIME; } - @Override - protected void assureSdConsistent() {} @Override public void getConfig(DocumentdbInfoConfig.Builder builder) { } + @Override public void getConfig(IndexInfoConfig.Builder builder) { } + @Override public void getConfig(IlscriptsConfig.Builder builder) { } + @Override public void getConfig(AttributesConfig.Builder builder) { } + @Override public void getConfig(RankProfilesConfig.Builder builder) { } + } public static AbstractSearchCluster mockSearchCluster(AbstractConfigProducerRoot root, String clusterName, int clusterIndex, boolean isStreaming) { diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index fee0edacb90..5df522e1b9d 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -29,8 +29,9 @@ public class NodeResourcesTuningTest { @Test public void require_that_hwinfo_memory_size_is_set() { - ProtonConfig cfg = configFromMemorySetting(24); - assertEquals(24 * GB, cfg.hwinfo().memory().size()); + double combinedFactor = 1 - 17.0/100; + assertEquals(24 * GB, configFromMemorySetting(24, false).hwinfo().memory().size()); + assertEquals(combinedFactor * 24 * GB, configFromMemorySetting(24, true).hwinfo().memory().size(), 1000); } private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb, int redundancy, int searchableCopies) { @@ -43,6 +44,7 @@ public class NodeResourcesTuningTest { } return configFromMemorySetting(gb, builder, redundancy, searchableCopies); } + private void verify_that_initial_numdocs_is_dependent_of_mode(int redundancy, int searchablecopies) { int divisor = Math.max(redundancy, searchablecopies); ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24, redundancy, searchablecopies); @@ -54,6 +56,7 @@ public class NodeResourcesTuningTest { assertEquals(402653184/divisor, cfg.documentdb(2).allocation().initialnumdocs()); assertEquals("c", cfg.documentdb(2).inputdoctypename()); } + @Test public void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { verify_that_initial_numdocs_is_dependent_of_mode(2,0); @@ -145,8 +148,15 @@ public class NodeResourcesTuningTest { @Test public void require_that_summary_cache_max_bytes_is_set_based_on_memory() { - assertEquals(1*GB/20, configFromMemorySetting(1).summary().cache().maxbytes()); - assertEquals(256*GB/20, configFromMemorySetting(256).summary().cache().maxbytes()); + assertEquals(1*GB / 20, configFromMemorySetting(1, false).summary().cache().maxbytes()); + assertEquals(256*GB / 20, configFromMemorySetting(256, false).summary().cache().maxbytes()); + } + + @Test + public void require_that_summary_cache_memory_is_reduced_with_combined_cluster() { + double combinedFactor = 1 - 17.0/100; + assertEquals(combinedFactor * 1*GB / 20, configFromMemorySetting(1, true).summary().cache().maxbytes(), 1000); + assertEquals(combinedFactor * 256*GB / 20, configFromMemorySetting(256, true).summary().cache().maxbytes(), 1000); } @Test @@ -163,12 +173,12 @@ public class NodeResourcesTuningTest { } private static void assertDocumentStoreMaxFileSize(long expFileSizeBytes, int memoryGb) { - assertEquals(expFileSizeBytes, configFromMemorySetting(memoryGb).summary().log().maxfilesize()); + assertEquals(expFileSizeBytes, configFromMemorySetting(memoryGb, false).summary().log().maxfilesize()); } private static void assertFlushStrategyMemory(long expMemoryBytes, int memoryGb) { - assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb).flush().memory().maxmemory()); - assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb).flush().memory().each().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb, false).flush().memory().maxmemory()); + assertEquals(expMemoryBytes, configFromMemorySetting(memoryGb, false).flush().memory().each().maxmemory()); } private static void assertFlushStrategyTlsSize(long expTlsSizeBytes, int diskGb) { @@ -188,56 +198,63 @@ public class NodeResourcesTuningTest { } private static void assertWriteFilter(double expMemoryLimit, int memoryGb) { - assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb).writefilter().memorylimit(), delta); + assertEquals(expMemoryLimit, configFromMemorySetting(memoryGb, false).writefilter().memorylimit(), delta); } private static ProtonConfig configFromDiskSetting(boolean fastDisk) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - fastDisk(fastDisk)); + return getConfig(new FlavorsConfig.Flavor.Builder().fastDisk(fastDisk), false); } private static ProtonConfig configFromDiskSetting(int diskGb) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - minDiskAvailableGb(diskGb)); + return getConfig(new FlavorsConfig.Flavor.Builder().minDiskAvailableGb(diskGb), false); } - private static ProtonConfig configFromMemorySetting(int memoryGb) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - minMainMemoryAvailableGb(memoryGb)); + private static ProtonConfig configFromMemorySetting(int memoryGb, boolean combined) { + return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), combined); } + private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder, int redundancy, int searchableCopies) { - return getConfig(new FlavorsConfig.Flavor.Builder(). - minMainMemoryAvailableGb(memoryGb), builder, redundancy, searchableCopies); + return getConfig(new FlavorsConfig.Flavor.Builder() + .minMainMemoryAvailableGb(memoryGb), builder, redundancy, searchableCopies, false); } private static ProtonConfig configFromNumCoresSetting(double numCores) { - return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores)); + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), false); } private static ProtonConfig configFromNumCoresSetting(double numCores, int numThreadsPerSearch) { - return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), new ProtonConfig.Builder(), 1, 1, numThreadsPerSearch); + return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores), + new ProtonConfig.Builder(), 1, 1, numThreadsPerSearch, false); } private static ProtonConfig configFromEnvironmentType(boolean docker) { String environment = (docker ? "DOCKER_CONTAINER" : "undefined"); - return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment)); + return getConfig(new FlavorsConfig.Flavor.Builder().environment(environment), false); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) { - return getConfig(flavorBuilder, new ProtonConfig.Builder()); + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, boolean combined) { + return getConfig(flavorBuilder, new ProtonConfig.Builder(), combined); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { - return getConfig(flavorBuilder, protonBuilder, 1,1); + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, + ProtonConfig.Builder protonBuilder, boolean combined) { + return getConfig(flavorBuilder, protonBuilder, 1, 1, combined); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies) { + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int redundancy, int searchableCopies, boolean combined) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), + redundancy, searchableCopies, 1, combined); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int redundancy, int searchableCopies, int numThreadsPerSearch) { + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int redundancy, int searchableCopies, int numThreadsPerSearch, boolean combined) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), redundancy, searchableCopies, numThreadsPerSearch); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), + redundancy, searchableCopies, numThreadsPerSearch, combined); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index 9e26fa9e657..9c69ba8f212 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -48,14 +48,14 @@ public class SearchNodeTest { } private static SearchNode createSearchNode(AbstractConfigProducer parent, String name, int distributionKey, - NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted) { - return SearchNode.create(parent, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted); + NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, boolean combined) { + return SearchNode.create(parent, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, Optional.empty(), Optional.empty(), isHosted, combined); } @Test public void requireThatBasedirIsCorrectForElasticMode() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); prepare(root, node); assertBaseDir(Defaults.getDefaults().underVespaHome("var/db/vespa/search/cluster.mycluster/n3"), node); } @@ -63,7 +63,7 @@ public class SearchNodeTest { @Test public void requireThatPreShutdownCommandIsEmptyWhenNotActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode", 3, new NodeSpec(7, 5), false, root.getDeployState().isHosted(), false); node.setHostResource(new HostResource(new Host(node, "mynbode"))); node.initService(root.deployLogger()); assertFalse(node.getPreShutdownCommand().isPresent()); @@ -72,7 +72,7 @@ public class SearchNodeTest { @Test public void requireThatPreShutdownCommandUsesPrepareRestartWhenActivated() { MockRoot root = new MockRoot(""); - SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted()); + SearchNode node = createSearchNode(root, "mynode2", 4, new NodeSpec(7, 5), true, root.getDeployState().isHosted(), false); node.setHostResource(new HostResource(new Host(node, "mynbode2"))); node.initService(root.deployLogger()); assertTrue(node.getPreShutdownCommand().isPresent()); |