diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-10-29 15:10:34 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-10-29 15:10:34 +0100 |
commit | 8f58f2bd6aceba7442fb0c79734c09186e5bbeb6 (patch) | |
tree | 63fed8403c025634ac70c54919e410f38c4e3409 /config-model/src | |
parent | f9076902965b87a25827a41925edda9b9e728f77 (diff) | |
parent | c6c9a282b21f7a9d88cdb64a9e60d2cfc03aafe4 (diff) |
Merge pull request #29135 from vespa-engine/balder/move-computation-of-target-num-docs-to-backend
Balder/move computation of target num docs to backend
Diffstat (limited to 'config-model/src')
7 files changed, 8 insertions, 120 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java index 273e5580403..490d58d84b8 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilder.java @@ -56,7 +56,7 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } else if (equals("flushstrategy", e)) { handleFlushStrategy(e, t.searchNode); } else if (equals("resizing", e)) { - handleResizing(deployLogger, e, t.searchNode); + deployLogger.logApplicationPackage(Level.WARNING, "searchnode.resizing is deprecated and ignored"); } else if (equals("index", e)) { handleIndex(deployLogger, e, t.searchNode); } else if (equals("attribute", e)) { @@ -164,18 +164,6 @@ public class DomSearchTuningBuilder extends VespaDomBuilder.DomConfigProducerBui } } - private void handleResizing(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { - sn.resizing = new Tuning.SearchNode.Resizing(); - for (Element e : XML.getChildren(spec)) { - if (equals("initialdocumentcount", e)) { - deployLogger.logApplicationPackage(Level.WARNING, "resizing.initialdocumentcount is deprecated."); - sn.resizing.initialDocumentCount = asInt(e); - } else if (equals("amortize-count", e)) { - deployLogger.logApplicationPackage(Level.WARNING, "resizing.amortize-count is deprecated and ignored"); - } - } - } - private void handleIndex(DeployLogger deployLogger, Element spec, Tuning.SearchNode sn) { sn.index = new Tuning.SearchNode.Index(); for (Element e : XML.getChildren(spec)) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java index 2beec421faa..2616dd8a93c 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeResourcesTuning.java @@ -23,21 +23,16 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { private final static double TLS_SIZE_FRACTION = 0.02; final static long MB = 1024 * 1024; public final static long GB = MB * 1024; - // This is an approximate number based on observation of a node using 33G memory with 765M docs - private final static long MEMORY_COST_PER_DOCUMENT_DB_ONLY = 46L; private final NodeResources resources; private final int threadsPerSearch; private final double fractionOfMemoryReserved; - private final Redundancy redundancy; public NodeResourcesTuning(NodeResources resources, int threadsPerSearch, - double fractionOfMemoryReserved, - Redundancy redundancy) { + double fractionOfMemoryReserved) { this.resources = resources; this.threadsPerSearch = threadsPerSearch; this.fractionOfMemoryReserved = fractionOfMemoryReserved; - this.redundancy = redundancy; } @Override @@ -52,18 +47,6 @@ public class NodeResourcesTuning implements ProtonConfig.Producer { tuneSummaryReadIo(builder.summary.read); tuneSummaryCache(builder.summary.cache); tuneSearchReadIo(builder.search.mmap); - for (ProtonConfig.Documentdb.Builder dbb : builder.documentdb) { - getConfig(dbb); - } - } - - private void getConfig(ProtonConfig.Documentdb.Builder builder) { - // TODO => Move this to backend to enable ignoring this setting. - ProtonConfig.Documentdb dbCfg = builder.build(); - if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)usableMemoryGb() * GB / MEMORY_COST_PER_DOCUMENT_DB_ONLY; - builder.allocation.initialnumdocs(numDocs/redundancy.readyCopies()); - } } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 83c1778b95e..c0f22da2982 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -284,7 +284,7 @@ public class SearchNode extends AbstractService implements if (nodeResources.isPresent()) { var nodeResourcesTuning = new NodeResourcesTuning(nodeResources.get(), tuning.map(Tuning::threadsPerSearch).orElse(1), - fractionOfMemoryReserved, redundancyProvider.redundancy()); + fractionOfMemoryReserved); nodeResourcesTuning.getConfig(builder); tuning.ifPresent(t -> t.getConfig(builder)); diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 9621ddd1374..90a2bd7c734 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -119,21 +119,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { } } - public static class Resizing implements ProtonConfig.Producer { - // TODO GC as soon as resource computation is moved to backend. - public Integer initialDocumentCount = null; - - @Override - public void getConfig(ProtonConfig.Builder builder) { - if (initialDocumentCount!=null) { - for (ProtonConfig.Documentdb.Builder db : builder.documentdb) { - db.allocation.initialnumdocs(initialDocumentCount); - } - } - } - - } - public static class Index implements ProtonConfig.Producer { public static class Io implements ProtonConfig.Producer { public IoType search = null; @@ -347,7 +332,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { public RequestThreads threads = null; public LidSpace lidSpace = null; public FlushStrategy strategy = null; - public Resizing resizing = null; public Index index = null; public Summary summary = null; public Initialize initialize = null; @@ -359,7 +343,6 @@ public class Tuning extends AnyConfigProducer implements ProtonConfig.Producer { if (threads != null) threads.getConfig(builder); if (lidSpace != null) lidSpace.getConfig(builder); if (strategy != null) strategy.getConfig(builder); - if (resizing != null) resizing.getConfig(builder); if (index != null) index.getConfig(builder); if (summary != null) summary.getConfig(builder); if (initialize != null) initialize.getConfig(builder); diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java index 963fce0c666..764e31fe13a 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSchemaTuningBuilderTest.java @@ -103,15 +103,6 @@ public class DomSchemaTuningBuilderTest extends DomBuilderTest { } @Test - void requireThatWeCanParseResizingTag() { - Tuning t = createTuning(parseXml("<resizing>", - "<initialdocumentcount>128</initialdocumentcount>", - "<amortize-count>13</amortize-count>", - "</resizing>")); - assertEquals(128, t.searchNode.resizing.initialDocumentCount.intValue()); - } - - @Test void requireThatWeCanParseIndexTag() { Tuning t = createTuning(parseXml("<index>", "<io>", "<search>mmap</search>", diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java index 1f143fd2d82..fbc4f6768a2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeResourcesTuningTest.java @@ -1,16 +1,12 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; -import com.yahoo.collections.Pair; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import com.yahoo.vespa.model.container.ApplicationContainerCluster; -import com.yahoo.vespa.model.content.Redundancy; import org.junit.jupiter.api.Test; -import java.util.List; - import static com.yahoo.vespa.model.Host.memoryOverheadGb; import static org.junit.jupiter.api.Assertions.assertEquals; import static com.yahoo.vespa.model.search.NodeResourcesTuning.MB; @@ -41,35 +37,6 @@ public class NodeResourcesTuningTest { assertEquals(0.7, memoryOverheadGb, delta); } - private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, double gb, Redundancy redundancy) { - ProtonConfig.Builder builder = new ProtonConfig.Builder(); - for (Pair<String, String> sdMode : sdAndMode) { - builder.documentdb.add(new ProtonConfig.Documentdb.Builder() - .inputdoctypename(sdMode.getFirst()) - .configid("some/config/id/" + sdMode.getFirst()) - .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond()))); - } - return configFromMemorySetting(gb, builder, redundancy); - } - - private void verify_that_initial_numdocs_is_dependent_of_mode(int readyCopies) { - ProtonConfig cfg = getProtonMemoryConfig(List.of(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), - 24 + memoryOverheadGb, new Redundancy(readyCopies+1,readyCopies+1, readyCopies,1, readyCopies)); - assertEquals(3, cfg.documentdb().size()); - assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); - assertEquals("a", cfg.documentdb(0).inputdoctypename()); - assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(1).allocation().initialnumdocs()); - assertEquals("b", cfg.documentdb(1).inputdoctypename()); - assertEquals(24 * GB / (46 * readyCopies), cfg.documentdb(2).allocation().initialnumdocs()); - assertEquals("c", cfg.documentdb(2).inputdoctypename()); - } - - @Test - void require_that_initial_numdocs_is_dependent_of_mode_and_searchablecopies() { - verify_that_initial_numdocs_is_dependent_of_mode(1); - verify_that_initial_numdocs_is_dependent_of_mode(2); - } - @Test void require_that_hwinfo_cpu_cores_is_set() { ProtonConfig cfg = configFromNumCoresSetting(24); @@ -227,9 +194,9 @@ public class NodeResourcesTuningTest { return getConfig(new FlavorsConfig.Flavor.Builder().minMainMemoryAvailableGb(memoryGb), fractionOfMemoryReserved); } - private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder, Redundancy redundancy) { + private static ProtonConfig configFromMemorySetting(double memoryGb, ProtonConfig.Builder builder) { return getConfig(new FlavorsConfig.Flavor.Builder() - .minMainMemoryAvailableGb(memoryGb), builder, redundancy); + .minMainMemoryAvailableGb(memoryGb), builder); } private static ProtonConfig configFromNumCoresSetting(double numCores) { @@ -255,11 +222,7 @@ public class NodeResourcesTuningTest { } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { - return getConfig(flavorBuilder, protonBuilder, new Redundancy(1,1,1,1,1)); - } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - Redundancy redundancy) { - return getConfig(flavorBuilder, protonBuilder,1, redundancy); + return getConfig(flavorBuilder, protonBuilder,1); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, double fractionOfMemoryReserved) { return getConfig(flavorBuilder, protonBuilder, 1, fractionOfMemoryReserved); @@ -267,24 +230,14 @@ public class NodeResourcesTuningTest { private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, new Redundancy(1,1,1,1,1)); - } - - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, Redundancy redundancy) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0, redundancy); + return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, 0); } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, int numThreadsPerSearch, double fractionOfMemoryReserved) { - return getConfig(flavorBuilder, protonBuilder, numThreadsPerSearch, fractionOfMemoryReserved, - new Redundancy(1,1,1,1,1)); - } - private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder, - int numThreadsPerSearch, double fractionOfMemoryReserved, Redundancy redundancy) { flavorBuilder.name("my_flavor"); NodeResourcesTuning tuning = new NodeResourcesTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder)).resources(), - numThreadsPerSearch, fractionOfMemoryReserved, redundancy); + numThreadsPerSearch, fractionOfMemoryReserved); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index 685610544fe..5b501bad876 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -173,16 +173,6 @@ public class DocumentDatabaseTestCase { "", List.of(DEFAULT, DEFAULT)); } - @Test - void requireThatMixedModeInitialDocumentCountIsReflected() { - final long INITIAL = 1000000000L; - String feedTuning = "<resizing>" + - " <initialdocumentcount>1000000000</initialdocumentcount>" + - "</resizing>\n"; - verifyInitialDocumentCount(List.of(DocType.create("a", "index"), DocType.create("b", "streaming")), - feedTuning, List.of(INITIAL, INITIAL)); - } - private void assertDocTypeConfig(VespaModel model, String configId, String indexField, String attributeField) { IndexschemaConfig icfg = model.getConfig(IndexschemaConfig.class, configId); assertEquals(1, icfg.indexfield().size()); |