diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-01-28 11:57:11 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2019-01-28 11:57:11 +0100 |
commit | 13ea12d795506ad51a41626be58be9c28c8a0234 (patch) | |
tree | cf62665902e9ed9512f7c83ff2c0189c58b53c8f /config-model | |
parent | 7ab7ef084c10547ad1e3456144d769c53849d7e0 (diff) | |
parent | 71722d38fc1bcf2e03a7856c862a048e33f17c2f (diff) |
Merge pull request #8241 from vespa-engine/balder/add-control-over-concurrency-and-allocation-per-documentdb
Balder/add control over concurrency and allocation per documentdb.
Diffstat (limited to 'config-model')
9 files changed, 219 insertions, 19 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index f4754b4f958..4c8a086bdcb 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -306,6 +306,8 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; + builder.feeding.concurrency(0.2); + boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); String docTypeName = type.getFullName().getName(); @@ -316,9 +318,26 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot .global(globalDocType); Optional<StreamingSearchCluster> ssc = findStreamingCluster(docTypeName); if (ssc.isPresent()) { - ddbB.inputdoctypename(type.getFullName().getName()).configid(ssc.get().getDocumentDBConfigId()); + hasAnyNonIndexedCluster = true; + ddbB.inputdoctypename(type.getFullName().getName()) + .configid(ssc.get().getDocumentDBConfigId()) + .mode(ProtonConfig.Documentdb.Mode.Enum.STREAMING) + .feeding.concurrency(0.0); } else if (hasIndexedCluster()) { - getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + if (getIndexed().hasDocumentDB(type.getFullName().getName())) { + getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); + if (tuning != null && tuning.searchNode != null && tuning.searchNode.feeding != null) { + ddbB.feeding.concurrency(tuning.searchNode.feeding.concurrency / 2); + } + } else { + hasAnyNonIndexedCluster = true; + ddbB.feeding.concurrency(0.0); + ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); + } + } else { + hasAnyNonIndexedCluster = true; + ddbB.feeding.concurrency(0.0); + ddbB.mode(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY); } if (globalDocType) { ddbB.visibilitydelay(0.0); @@ -339,6 +358,9 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot if (redundancy != null) { redundancy.getConfig(builder); } + if (hasAnyNonIndexedCluster) { + builder.feeding.concurrency(builder.feeding.build().concurrency() * 2); + } } private boolean isGloballyDistributed(NewDocumentType docType) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java index 80bba10276d..2d11f868f04 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/IndexedSearchCluster.java @@ -309,6 +309,14 @@ public class IndexedSearchCluster extends SearchCluster public List<DocumentDatabase> getDocumentDbs() { return documentDbs; } + public boolean hasDocumentDB(String name) { + for (DocumentDatabase db : documentDbs) { + if (db.getName().equals(name)) { + return true; + } + } + return false; + } public void setSearchCoverage(SearchCoverage searchCoverage) { this.searchCoverage = searchCoverage; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index b845b180548..8cf1db3d6f5 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -31,6 +31,17 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { tuneSummaryReadIo(builder.summary.read); tuneSummaryCache(builder.summary.cache); tuneSearchReadIo(builder.search.mmap); + for (ProtonConfig.Documentdb.Builder dbb : builder.documentdb) { + getConfig(dbb); + } + } + + private void getConfig(ProtonConfig.Documentdb.Builder builder) { + ProtonConfig.Documentdb dbCfg = builder.build(); + if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { + long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/40L; + builder.allocation.initialnumdocs(numDocs); + } } private void tuneSummaryCache(ProtonConfig.Summary.Cache.Builder builder) { diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java index 5dcf60e5fac..5be51310504 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/SearchNode.java @@ -4,6 +4,7 @@ package com.yahoo.vespa.model.search; import com.yahoo.cloud.config.filedistribution.FiledistributorrpcConfig; import com.yahoo.config.model.deploy.DeployState; import com.yahoo.config.model.producer.AbstractConfigProducer; +import com.yahoo.config.provision.Flavor; import com.yahoo.metrics.MetricsmanagerConfig; import com.yahoo.searchlib.TranslogserverConfig; import com.yahoo.vespa.config.content.LoadTypeConfig; @@ -260,7 +261,10 @@ public class SearchNode extends AbstractService implements builder.pruneremoveddocumentsage(4 * 24 * 3600 + 3600 + 60); } if (getHostResource() != null && getHostResource().getFlavor().isPresent()) { - new NodeFlavorTuning(getHostResource().getFlavor().get()).getConfig(builder); + Flavor nodeFlavor = getHostResource().getFlavor().get(); + NodeFlavorTuning nodeFlavorTuning = new NodeFlavorTuning(nodeFlavor); + nodeFlavorTuning.getConfig(builder); + if (tuning.isPresent()) { tuning.get().getConfig(builder); } diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java index 8e8192b74fa..a87a3d3e312 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/Tuning.java @@ -134,6 +134,9 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P public void getConfig(ProtonConfig.Builder builder) { if (initialDocumentCount!=null) { builder.grow.initial(initialDocumentCount); + for (ProtonConfig.Documentdb.Builder db : builder.documentdb) { + db.allocation.initialnumdocs(initialDocumentCount); + } } } @@ -338,7 +341,9 @@ public class Tuning extends AbstractConfigProducer implements PartitionsConfig.P @Override public void getConfig(ProtonConfig.Builder builder) { if (concurrency != null) { - builder.feeding.concurrency(concurrency); + // We divide by 2 as this number is used for 2 different thread pools. + // Not perfect, but the best way to split the resources evenly. + builder.feeding.concurrency(concurrency/2); } } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java index 145715bdc16..aa52a2b586e 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/builder/xml/dom/DomSearchTuningBuilderTest.java @@ -255,7 +255,7 @@ public class DomSearchTuningBuilderTest extends DomBuilderTest { "<concurrency>0.7</concurrency>", "</feeding>")); assertEquals(0.7, t.searchNode.feeding.concurrency.doubleValue(), DELTA); - assertThat(getProtonCfg(t), containsString("feeding.concurrency 0.7")); + assertThat(getProtonCfg(t), containsString("feeding.concurrency 0.35")); } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java index cc5ec34b43e..ba10634081f 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/content/utils/DocType.java @@ -25,7 +25,10 @@ public class DocType { return (global ? "<document mode='" + mode + "' type='" + type + "' global='true'/>" : "<document mode='" + mode + "' type='" + type + "'/>"); } + public String getType() { return type; } + public static DocType create(String type, String mode) { return new DocType(type, mode, false); } + public static DocType createGlobal(String type, String mode) { return new DocType(type, mode, true); } public static DocType storeOnly(String type) { return new DocType(type, "store-only", false); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index 95503550767..56d53c2cc69 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -1,12 +1,17 @@ // Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.vespa.model.search; +import com.yahoo.collections.Pair; import com.yahoo.config.provision.Flavor; import com.yahoo.config.provisioning.FlavorsConfig; import com.yahoo.vespa.config.search.core.ProtonConfig; import org.junit.Test; +import java.util.Arrays; +import java.util.List; + import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; import static com.yahoo.vespa.model.search.NodeFlavorTuning.MB; import static com.yahoo.vespa.model.search.NodeFlavorTuning.GB; @@ -27,6 +32,29 @@ public class NodeFlavorTuningTest { assertEquals(24 * GB, cfg.hwinfo().memory().size()); } + private ProtonConfig getProtonMemoryConfig(List<Pair<String, String>> sdAndMode, int gb) { + ProtonConfig.Builder builder = new ProtonConfig.Builder(); + for (Pair<String, String> sdMode : sdAndMode) { + builder.documentdb.add(new ProtonConfig.Documentdb.Builder() + .inputdoctypename(sdMode.getFirst()) + .configid("some/config/id/" + sdMode.getFirst()) + .mode(ProtonConfig.Documentdb.Mode.Enum.valueOf(sdMode.getSecond()))); + } + return configFromMemorySetting(gb, builder); + } + @Test + public void require_that_initial_numdocs_is_dependent_of_mode() { + ProtonConfig cfg = getProtonMemoryConfig(Arrays.asList(new Pair<>("a", "INDEX"), new Pair<>("b", "STREAMING"), new Pair<>("c", "STORE_ONLY")), 24); + assertEquals(1024, cfg.grow().initial()); + assertEquals(3, cfg.documentdb().size()); + assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); + assertEquals("a", cfg.documentdb(0).inputdoctypename()); + assertEquals(644245094L, cfg.documentdb(1).allocation().initialnumdocs()); + assertEquals("b", cfg.documentdb(1).inputdoctypename()); + assertEquals(644245094L, cfg.documentdb(2).allocation().initialnumdocs()); + assertEquals("c", cfg.documentdb(2).inputdoctypename()); + } + @Test public void require_that_hwinfo_cpu_cores_is_set() { ProtonConfig cfg = configFromNumCoresSetting(24); @@ -143,6 +171,10 @@ public class NodeFlavorTuningTest { return getConfig(new FlavorsConfig.Flavor.Builder(). minMainMemoryAvailableGb(memoryGb)); } + private static ProtonConfig configFromMemorySetting(int memoryGb, ProtonConfig.Builder builder) { + return getConfig(new FlavorsConfig.Flavor.Builder(). + minMainMemoryAvailableGb(memoryGb), builder); + } private static ProtonConfig configFromNumCoresSetting(double numCores) { return getConfig(new FlavorsConfig.Flavor.Builder().minCpuCores(numCores)); @@ -154,11 +186,18 @@ public class NodeFlavorTuningTest { } private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder) { + getConfig(flavorBuilder, new ProtonConfig.Builder()); flavorBuilder.name("my_flavor"); NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))); ProtonConfig.Builder protonBuilder = new ProtonConfig.Builder(); tuning.getConfig(protonBuilder); return new ProtonConfig(protonBuilder); } + private static ProtonConfig getConfig(FlavorsConfig.Flavor.Builder flavorBuilder, ProtonConfig.Builder protonBuilder) { + flavorBuilder.name("my_flavor"); + NodeFlavorTuning tuning = new NodeFlavorTuning(new Flavor(new FlavorsConfig.Flavor(flavorBuilder))); + tuning.getConfig(protonBuilder); + return new ProtonConfig(protonBuilder); + } } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index bd27f381e4e..ec10ffa68a2 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -12,13 +12,14 @@ import com.yahoo.vespa.config.search.AttributesConfig; import com.yahoo.vespa.configdefinition.IlscriptsConfig; import com.yahoo.vespa.model.VespaModel; import com.yahoo.vespa.model.content.ContentSearchCluster; +import com.yahoo.vespa.model.content.utils.DocType; import com.yahoo.vespa.model.search.IndexedSearchCluster; import com.yahoo.vespa.model.test.utils.ApplicationPackageUtils; import com.yahoo.vespa.model.test.utils.VespaModelCreatorWithMockPkg; import org.junit.Test; import org.xml.sax.SAXException; import java.io.IOException; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Arrays; import java.util.Map; @@ -31,6 +32,8 @@ import static org.junit.Assert.assertEquals; */ public class DocumentDatabaseTestCase { + private static final double SMALL = 0.00000000000001; + private String vespaHosts = "<?xml version='1.0' encoding='utf-8' ?>" + "<hosts> " + " <host name='foo'>" + @@ -38,7 +41,14 @@ public class DocumentDatabaseTestCase { " </host>" + "</hosts>"; - private String createVespaServices(List<String> sdNames, String mode) { + private String createVespaServices(List<String> sds, String mode) { + List<DocType> nameAndModes = new ArrayList<>(sds.size()); + for (String sd : sds) { + nameAndModes.add(DocType.create(sd, mode)); + } + return createVespaServicesXml(nameAndModes, ""); + } + private String createVespaServicesXml(List<DocType> nameAndModes, String xmlTuning) { StringBuilder retval = new StringBuilder(); retval.append("" + "<?xml version='1.0' encoding='utf-8' ?>\n" + @@ -54,18 +64,22 @@ public class DocumentDatabaseTestCase { "</container>\n" + "<content version='1.0' id='test'>\n" + " <redundancy>1</redundancy>\n"); - retval.append(" <documents>\n"); - for (String sdName : sdNames) { - retval.append("").append(" <document type='").append(sdName).append("' mode='").append(mode).append("'"); - retval.append("/>\n"); - } - retval.append(" </documents>\n"); - retval.append("" + - " <nodes>\n" + - " <node hostalias='node0' distribution-key='0'/>\n" + - " </nodes>\n" + - "</content>\n" + - "</services>\n"); + retval.append(DocType.listToXml(nameAndModes)); + retval.append( + " <engine>\n" + + " <proton>\n" + + " <tuning>\n" + + " <searchnode>\n" + + xmlTuning + + " </searchnode>\n" + + " </tuning\n>" + + " </proton\n>" + + " </engine\n>" + + " <nodes>\n" + + " <node hostalias='node0' distribution-key='0'/>\n" + + " </nodes>\n" + + " </content>\n" + + "</services>\n"); return retval.toString(); } @@ -93,6 +107,100 @@ public class DocumentDatabaseTestCase { assertSingleSD("index"); } + private VespaModel createModel(List<DocType> nameAndModes, String xmlTuning) { + List<String> sds = new ArrayList<>(nameAndModes.size()); + for (DocType nameAndMode : nameAndModes) { + sds.add(nameAndMode.getType()); + } + return new VespaModelCreatorWithMockPkg(vespaHosts, createVespaServicesXml(nameAndModes, xmlTuning), + ApplicationPackageUtils.generateSearchDefinitions(sds)).create(); + } + + @Test + public void requireThatConcurrencyIsReflectedCorrectlyForDefault() { + verifyConcurrency("index", "", 0.2, 0.2); + verifyConcurrency("streaming", "", 0.4, 0.0); + verifyConcurrency("store-only", "", 0.4, 0.0); + } + @Test + public void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 0.4, Arrays.asList(0.2, 0.0)); + } + @Test + public void requireThatMixedModeConcurrencyIsReflected() { + String feedTuning = "<feeding>" + + " <concurrency>0.7</concurrency>" + + "</feeding>\n"; + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), feedTuning, 0.7, Arrays.asList(0.35, 0.0)); + } + @Test + public void requireThatConcurrencyIsReflected() { + String feedTuning = "<feeding>" + + " <concurrency>0.7</concurrency>" + + "</feeding>\n"; + verifyConcurrency("index", feedTuning, 0.35, 0.35); + verifyConcurrency("streaming", feedTuning, 0.7, 0.0); + verifyConcurrency("store-only", feedTuning, 0.7, 0.0); + } + private void verifyConcurrency(String mode, String xmlTuning, double global, double local) { + verifyConcurrency(Arrays.asList(DocType.create("a", mode)), xmlTuning, global, Arrays.asList(local)); + } + private void verifyConcurrency(List<DocType> nameAndModes, String xmlTuning, double global, List<Double> local) { + assertEquals(nameAndModes.size(), local.size()); + VespaModel model = createModel(nameAndModes, xmlTuning); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(global, proton.feeding().concurrency(), SMALL); + assertEquals(local.size(), proton.documentdb().size()); + for (int i = 0; i < local.size(); i++) { + assertEquals(local.get(i), proton.documentdb(i).feeding().concurrency(), SMALL); + } + } + + @Test + public void requireThatModeIsSet() { + VespaModel model = createModel(Arrays.asList(DocType.create("a", "index"), + DocType.create("b", "streaming"), + DocType.create("c", "store-only")), ""); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(3, proton.documentdb().size()); + assertEquals(ProtonConfig.Documentdb.Mode.Enum.INDEX, proton.documentdb(0).mode()); + assertEquals("a", proton.documentdb(0).inputdoctypename()); + assertEquals(ProtonConfig.Documentdb.Mode.Enum.STREAMING, proton.documentdb(1).mode()); + assertEquals("b", proton.documentdb(1).inputdoctypename()); + assertEquals(ProtonConfig.Documentdb.Mode.Enum.STORE_ONLY, proton.documentdb(2).mode()); + assertEquals("c", proton.documentdb(2).inputdoctypename()); + } + + private void verifyInitialDocumentCount(List<DocType> nameAndModes, String xmlTuning, long global, List<Long> local) { + assertEquals(nameAndModes.size(), local.size()); + VespaModel model = createModel(nameAndModes, xmlTuning); + ContentSearchCluster contentSearchCluster = model.getContentClusters().get("test").getSearch(); + ProtonConfig proton = getProtonCfg(contentSearchCluster); + assertEquals(global, proton.grow().initial()); + assertEquals(local.size(), proton.documentdb().size()); + for (int i = 0; i < local.size(); i++) { + assertEquals(local.get(i).longValue(), proton.documentdb(i).allocation().initialnumdocs()); + } + } + + @Test + public void requireThatMixedModeInitialDocumentCountIsReflectedCorrectlyForDefault() { + final long DEFAULT = 1024L; + verifyInitialDocumentCount(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), + "", DEFAULT, Arrays.asList(DEFAULT, DEFAULT)); + } + @Test + public void requireThatMixedModeInitialDocumentCountIsReflected() { + final long INITIAL = 1000000000L; + String feedTuning = "<resizing>" + + " <initialdocumentcount>1000000000</initialdocumentcount>" + + "</resizing>\n"; + verifyInitialDocumentCount(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), + feedTuning, INITIAL, Arrays.asList(INITIAL, INITIAL)); + } + private void assertDocTypeConfig(VespaModel model, String configId, String indexField, String attributeField) { IndexschemaConfig icfg = model.getConfig(IndexschemaConfig.class, configId); assertEquals(1, icfg.indexfield().size()); |