From f553e748b93d7492c0743e508c3d00c27d091500 Mon Sep 17 00:00:00 2001 From: Arnstein Ressem Date: Tue, 1 Aug 2023 18:17:51 +0200 Subject: Revert "- initial document count must be number of unique documents per node." --- .../vespa/model/content/ContentSearchCluster.java | 4 +- .../vespa/model/search/NodeResourcesTuning.java | 8 +--- .../com/yahoo/vespa/model/search/SearchNode.java | 23 ++++------- .../model/search/NodeResourcesTuningTest.java | 45 ++++++++-------------- .../vespa/model/search/test/SearchNodeTest.java | 3 +- 5 files changed, 28 insertions(+), 55 deletions(-) diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index bf8c96db614..34ea41384bc 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -303,7 +303,7 @@ public class ContentSearchCluster extends TreeConfigProducer if (element == null) { searchNode = SearchNode.create(parent, "" + node.getDistributionKey(), node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, redundancy, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags()); searchNode.setHostResource(node.getHostResource()); searchNode.initService(deployState); @@ -312,7 +312,7 @@ public class ContentSearchCluster extends TreeConfigProducer tls.initService(deployState); } else { searchNode = new SearchNode.Builder(""+node.getDistributionKey(), spec, clusterName, node, flushOnShutdown, - tuning, resourceLimits, fractionOfMemoryReserved, redundancy) + tuning, resourceLimits, fractionOfMemoryReserved) .build(deployState, parent, element.getXml()); tls = new TransactionLogServer.Builder(clusterName, syncTransactionLog).build(deployState, searchNode, element.getXml()); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 02f15ed06cd..1ad99404823 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -4,7 +4,6 @@ package com.yahoo.vespa.model.search; import com.yahoo.config.provision.NodeResources; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.Host; -import com.yahoo.vespa.model.content.Redundancy; import static java.lang.Long.min; import static java.lang.Long.max; @@ -28,16 +27,13 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; - private final Redundancy redundancy; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - double fractionOfMemoryReserved, - Redundancy redundancy) { + double fractionOfMemoryReserved) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.redundancy = redundancy; } @Override @@ -61,7 +57,7 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_STORE_ONLY; - builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies()); + builder.allocation.initialnumdocs(numDocs); } } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index f238d30308b..c3de655af92 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -21,7 +21,6 @@ import com.yahoo.vespa.model.admin.monitoring.Monitoring; import com.yahoo.vespa.model.application.validation.RestartConfigs; import com.yahoo.vespa.model.builder.xml.dom.VespaDomBuilder; import com.yahoo.vespa.model.content.ContentNode; -import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.content.ResourceLimits; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProducer; import com.yahoo.vespa.model.filedistribution.FileDistributionConfigProvider; @@ -70,7 +69,6 @@ public class SearchNode extends AbstractService implements private final Optional tuning; private final Optional resourceLimits; private final double fractionOfMemoryReserved; - private final Redundancy redundancy; public static class Builder extends VespaDomBuilder.DomConfigProducerBuilderBase { @@ -82,11 +80,10 @@ public class SearchNode extends AbstractService implements private final Optional tuning; private final Optional resourceLimits; private final double fractionOfMemoryReserved; - private final Redundancy redundancy; public Builder(String name, NodeSpec nodeSpec, String clusterName, ContentNode node, boolean flushOnShutdown, Optional tuning, Optional resourceLimits, - double fractionOfMemoryReserved, Redundancy redundancy) { + double fractionOfMemoryReserved) { this.name = name; this.nodeSpec = nodeSpec; this.clusterName = clusterName; @@ -95,26 +92,23 @@ public class SearchNode extends AbstractService implements this.tuning = tuning; this.resourceLimits = resourceLimits; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.redundancy = redundancy; } @Override - protected SearchNode doBuild(DeployState deployState, TreeConfigProducer ancestor, - Element producerSpec) { + protected SearchNode doBuild(DeployState deployState, TreeConfigProducer ancestor, Element producerSpec) { return SearchNode.create(ancestor, name, contentNode.getDistributionKey(), nodeSpec, clusterName, contentNode, flushOnShutdown, tuning, resourceLimits, deployState.isHosted(), - fractionOfMemoryReserved, redundancy, deployState.featureFlags()); + fractionOfMemoryReserved, deployState.featureFlags()); } } public static SearchNode create(TreeConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, - Optional tuning, Optional resourceLimits, - boolean isHostedVespa, double fractionOfMemoryReserved, Redundancy redundancy, - ModelContext.FeatureFlags featureFlags) { + Optional tuning, Optional resourceLimits, boolean isHostedVespa, + double fractionOfMemoryReserved, ModelContext.FeatureFlags featureFlags) { SearchNode node = new SearchNode(parent, name, distributionKey, nodeSpec, clusterName, serviceLayerService, flushOnShutdown, - tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved, redundancy); + tuning, resourceLimits, isHostedVespa, fractionOfMemoryReserved); if (featureFlags.loadCodeAsHugePages()) { node.addEnvironmentVariable("VESPA_LOAD_CODE_AS_HUGEPAGES", true); } @@ -127,7 +121,7 @@ public class SearchNode extends AbstractService implements private SearchNode(TreeConfigProducer parent, String name, int distributionKey, NodeSpec nodeSpec, String clusterName, AbstractService serviceLayerService, boolean flushOnShutdown, Optional tuning, Optional resourceLimits, boolean isHostedVespa, - double fractionOfMemoryReserved, Redundancy redundancy) { + double fractionOfMemoryReserved) { super(parent, name); this.distributionKey = distributionKey; this.serviceLayerService = serviceLayerService; @@ -144,7 +138,6 @@ public class SearchNode extends AbstractService implements // Properties are set in DomSearchBuilder this.tuning = tuning; this.resourceLimits = resourceLimits; - this.redundancy = redundancy; setPropertiesElastic(clusterName, distributionKey); addEnvironmentVariable("OMP_NUM_THREADS", 1); } @@ -286,7 +279,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved, redundancy); + fractionOfMemoryReserved); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index d5215ab5752..8e719fa90c3 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -6,9 +6,9 @@ import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.content.Redundancy; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.List; import static com.yahoo.vespa.model.Host.memoryOverheadGb; @@ -22,6 +22,7 @@ import static com.yahoo.vespa.model.search.NodeResourcesTuning.GB; public class NodeResourcesTuningTest { private static final double delta = 0.00001; + private static final double combinedFactor = 1 - 18.0/100; private static final double DEFAULT_MEMORY_GAIN = 0.08; @Test @@ -41,7 +42,7 @@ public class NodeResourcesTuningTest { assertEquals(0.7, memoryOverheadGb, delta); } - private ProtonConfig getProtonMemoryConfig(List> sdAndMode, double gb, Redundancy redundancy) { + private ProtonConfig getProtonMemoryConfig(List> sdAndMode, double gb) { ProtonConfig.Builder builder = new ProtonConfig.Builder(); for (Pair sdMode : sdAndMode) { builder.documentdb.add(new ProtonConfig.Documentdb.Builder() @@ -49,25 +50,24 @@ public class NodeResourcesTuningTest { .configid("some/config/id/" + sdMode.getFirst()) .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond()))); } - return configFromMemorySetting(gb, builder, redundancy); + return configFromMemorySetting(gb, builder); } - private void verify_that_initial_numdocs_is_dependent_of_mode(int readyCopies) { - ProtonConfig cfg = getProtonMemoryConfig(List.of(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), - 24 + memoryOverheadGb, new Redundancy(readyCopies+1,readyCopies+1, readyCopies,1, readyCopies)); + private void verify_that_initial_numdocs_is_dependent_of_mode() { + ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24 + memoryOverheadGb); assertEquals(3, cfg.documentdb().size()); assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); assertEquals("a", cfg.documentdb(0).inputdoctypename()); - assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(1).allocation().initialnumdocs()); + assertEquals(24 * GB / 46, cfg.documentdb(1).allocation().initialnumdocs()); assertEquals("b", cfg.documentdb(1).inputdoctypename()); - assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(2).allocation().initialnumdocs()); + assertEquals(24 * GB / 46, cfg.documentdb(2).allocation().initialnumdocs()); assertEquals("c", cfg.documentdb(2).inputdoctypename()); } @Test void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { - verify_that_initial_numdocs_is_dependent_of_mode(1); - verify_that_initial_numdocs_is_dependent_of_mode(2); + verify_that_initial_numdocs_is_dependent_of_mode(); + } @Test @@ -227,9 +227,9 @@ public class NodeResourcesTuningTest { return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved); } - private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder, Redundancy redundancy) { + private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { return getConfig(new FlavorsConfig.Flavor.Builder() - .minMainMemoryAvailableGb(memoryGb), builder, redundancy); + .minMainMemoryAvailableGb(memoryGb), builder); } private static ProtonConfig configFromNumCoresSetting(double numCores) { @@ -255,11 +255,7 @@ public class NodeResourcesTuningTest { } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { - return getConfig(flavorBuilder, protonBuilder, new Redundancy(1,1,1,1,1)); - } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - Redundancy redundancy) { - return getConfig(flavorBuilder, protonBuilder,1, redundancy); + return getConfig(flavorBuilder, protonBuilder,1); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) { return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved); @@ -267,24 +263,13 @@ public class NodeResourcesTuningTest { private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, new Redundancy(1,1,1,1,1)); - } - - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, Redundancy redundancy) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, redundancy); + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch, double fractionOfMemoryReserved) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, fractionOfMemoryReserved, - new Redundancy(1,1,1,1,1)); - } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, double fractionOfMemoryReserved, Redundancy redundancy) { flavorBuilder.name("my_flavor"); - NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), - numThreadsPerSearch, fractionOfMemoryReserved, redundancy); + NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), numThreadsPerSearch, fractionOfMemoryReserved); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java index b357a3e6718..f60441bac03 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/SearchNodeTest.java @@ -10,7 +10,6 @@ import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.defaults.Defaults; import com.yahoo.vespa.model.Host; import com.yahoo.vespa.model.HostResource; -import com.yahoo.vespa.model.content.Redundancy; import com.yahoo.vespa.model.search.NodeSpec; import com.yahoo.vespa.model.search.SearchNode; import com.yahoo.vespa.model.search.TransactionLogServer; @@ -50,7 +49,7 @@ public class SearchNodeTest { private static SearchNode createSearchNode(MockRoot root, String name, int distributionKey, NodeSpec nodeSpec, boolean flushOnShutDown, boolean isHosted, ModelContext.FeatureFlags featureFlags) { return SearchNode.create(root, name, distributionKey, nodeSpec, "mycluster", null, flushOnShutDown, - Optional.empty(), Optional.empty(), isHosted, 0.0, new Redundancy(1,1,1,1,1), featureFlags); + Optional.empty(), Optional.empty(), isHosted, 0.0, featureFlags); } private static SearchNode createSearchNode(MockRoot root) { -- cgit v1.2.3