diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-08-02 09:12:25 +0200 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-08-02 12:04:34 +0200 |
commit | cc83212d4c9e1bce12977cdd6d7ff2049c3faa45 (patch) | |
tree | 95874d7a8ec74cbca0b7c8d7f0e868d82e8e2750 /config-model | |
parent | ef8786173aacd4cb9f12b05ce1eacce0bcc39a76 (diff) |
Revert "Revert "- initial document count must be number of unique documents per node." MERGEOK"
Diffstat (limited to 'config-model')
5 files changed, 55 insertions, 28 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 3eb56096e18..0b07ef59410 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -303,7 +303,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, redundancy, deployState.featureFlags()); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); @@ -312,7 +312,7 @@ public class ContentSearchCluster extends TreeConfigProducer<AnyConfigProducer> tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved) + tuning, resourceLimits, fractionOfMemoryReserved, redundancy) .build(deployState, parent, element.getXml()); tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 1ad99404823..02f15ed06cd 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.Host; +import com.yahoo.vespa.model.content.Redundancy; import static java.lang.Long.min; import static java.lang.Long.max; @@ -27,13 +28,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; + private final Redundancy redundancy; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, + Redundancy redundancy) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.redundancy = redundancy; } @Override @@ -57,7 +61,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY; - builder.allocation.initialnumdocs(numDocs); + builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies()); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index c3de655af92..f238d30308b 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -21,6 +21,7 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.ContentNode; +import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ResourceLimits; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; @@ -69,6 +70,7 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; + private final Redundancy redundancy; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase<SearchNode> { @@ -80,10 +82,11 @@ public class SearchNode extends AbstractService implements private final Optional<Tuning> tuning; private final Optional<ResourceLimits> resourceLimits; private final double fractionOfMemoryReserved; + private final Redundancy redundancy; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Redundancy redundancy) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -92,23 +95,26 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; + this.redundancy = redundancy; } @Override - protected SearchNode doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, Element producerSpec) { + protected SearchNode doBuild(DeployState deployState, TreeConfigProducer<AnyConfigProducer> ancestor, + Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, deployState.featureFlags()); + fractionOfMemoryReserved, redundancy, deployState.featureFlags()); } } public static SearchNode create(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, ModelContext.FeatureFlags featureFlags) { + Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, + boolean isHostedVespa, double fractionOfMemoryReserved, Redundancy redundancy, + ModelContext.FeatureFlags featureFlags) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, redundancy); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -121,7 +127,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer<?> parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional<Tuning> tuning, Optional<ResourceLimits> resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved) { + double fractionOfMemoryReserved, Redundancy redundancy) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -138,6 +144,7 @@ public class SearchNode extends AbstractService implements // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; + this.redundancy = redundancy; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -279,7 +286,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved); + fractionOfMemoryReserved, redundancy); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 8e719fa90c3..d5215ab5752 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -6,9 +6,9 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; +import com.yahoo.vespa.model.content.Redundancy; import org.junit.jupiter.api.Test; -import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.model.Host.memoryOverheadGb; @@ -22,7 +22,6 @@ import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; public class NodeResourcesTuningTest { private static final double delta = 0.00001; - private static final double combinedFactor = 1 - 18.0/100; private static final double DEFAULT_MEMORY_GAIN = 0.08; @Test @@ -42,7 +41,7 @@ public class NodeResourcesTuningTest { assertEquals(0.7, memoryOverheadGb, delta); } - private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb) { + private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb, Redundancy redundancy) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); for (Pair<String, String> sdMode : sdAndMode) { builder.documentdb.add(new ProtonConfig.Documentdb.Builder() @@ -50,24 +49,25 @@ public class NodeResourcesTuningTest { .configid("some/config/id/" + sdMode.getFirst()) .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond()))); } - return configFromMemorySetting(gb, builder); + return configFromMemorySetting(gb, builder, redundancy); } - private void verify_that_initial_numdocs_is_dependent_of_mode() { - ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + memoryOverheadGb); + private void verify_that_initial_numdocs_is_dependent_of_mode(int readyCopies) { + ProtonConfig cfg = getProtonMemoryConfig(List.of(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), + 24 + memoryOverheadGb, new Redundancy(readyCopies+1,readyCopies+1, readyCopies,1, readyCopies)); assertEquals(3, cfg.documentdb().size()); assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); assertEquals("a", cfg.documentdb(0).inputdoctypename()); - assertEquals(24 * GB / 46, cfg.documentdb(1).allocation().initialnumdocs()); + assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(1).allocation().initialnumdocs()); assertEquals("b", cfg.documentdb(1).inputdoctypename()); - assertEquals(24 * GB / 46, cfg.documentdb(2).allocation().initialnumdocs()); + assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(2).allocation().initialnumdocs()); assertEquals("c", cfg.documentdb(2).inputdoctypename()); } @Test void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { - verify_that_initial_numdocs_is_dependent_of_mode(); - + verify_that_initial_numdocs_is_dependent_of_mode(1); + verify_that_initial_numdocs_is_dependent_of_mode(2); } @Test @@ -227,9 +227,9 @@ public class NodeResourcesTuningTest { return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved); } - private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { + private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder, Redundancy redundancy) { return getConfig(new FlavorsConfig.Flavor.Builder() - .minMainMemoryAvailableGb(memoryGb), builder); + .minMainMemoryAvailableGb(memoryGb), builder, redundancy); } private static ProtonConfig configFromNumCoresSetting(double numCores) { @@ -255,7 +255,11 @@ public class NodeResourcesTuningTest { } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { - return getConfig(flavorBuilder, protonBuilder,1); + return getConfig(flavorBuilder, protonBuilder, new Redundancy(1,1,1,1,1)); + } + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + Redundancy redundancy) { + return getConfig(flavorBuilder, protonBuilder,1, redundancy); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) { return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved); @@ -263,13 +267,24 @@ public class NodeResourcesTuningTest { private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0); + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, new Redundancy(1,1,1,1,1)); + } + + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int numThreadsPerSearch, Redundancy redundancy) { + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, redundancy); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch, double fractionOfMemoryReserved) { + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, fractionOfMemoryReserved, + new Redundancy(1,1,1,1,1)); + } + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, + int numThreadsPerSearch, double fractionOfMemoryReserved, Redundancy redundancy) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), + numThreadsPerSearch, fractionOfMemoryReserved, redundancy); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index f60441bac03..b357a3e6718 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -10,6 +10,7 @@ import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; +import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.TransactionLogServer; @@ -49,7 +50,7 @@ public class SearchNodeTest { private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - Optional.empty(), Optional.empty(), isHosted, 0.0, featureFlags); + Optional.empty(), Optional.empty(), isHosted, 0.0, new Redundancy(1,1,1,1,1), featureFlags); } private static SearchNode createSearchNode(MockRoot root) { |