diff options
25 files changed, 70 insertions, 89 deletions
diff --git a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java index 7900ce9ffec..219c8a01afe 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/content/ContentSearchCluster.java @@ -306,7 +306,7 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot @Override public void getConfig(ProtonConfig.Builder builder) { double visibilityDelay = hasIndexedCluster() ? getIndexed().getVisibilityDelay() : 0.0; - builder.feeding.concurrency(0.2); + builder.feeding.concurrency(0.25); // As if specified 0.5 in services.xml boolean hasAnyNonIndexedCluster = false; for (NewDocumentType type : TopologicalDocumentTypeSorter.sort(documentDefinitions.values())) { ProtonConfig.Documentdb.Builder ddbB = new ProtonConfig.Documentdb.Builder(); @@ -328,6 +328,8 @@ public class ContentSearchCluster extends AbstractConfigProducer implements Prot getIndexed().fillDocumentDBConfig(type.getFullName().getName(), ddbB); if (tuning != null && tuning.searchNode != null && tuning.searchNode.feeding != null) { ddbB.feeding.concurrency(tuning.searchNode.feeding.concurrency / 2); + } else { + ddbB.feeding.concurrency(builder.feeding.build().concurrency()); } } else { hasAnyNonIndexedCluster = true; diff --git a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java index 8cf1db3d6f5..da193a17801 100644 --- a/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java +++ b/config-model/src/main/java/com/yahoo/vespa/model/search/NodeFlavorTuning.java @@ -39,7 +39,7 @@ public class NodeFlavorTuning implements ProtonConfig.Producer { private void getConfig(ProtonConfig.Documentdb.Builder builder) { ProtonConfig.Documentdb dbCfg = builder.build(); if (dbCfg.mode() != ProtonConfig.Documentdb.Mode.Enum.INDEX) { - long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/40L; + long numDocs = (long)nodeFlavor.getMinMainMemoryAvailableGb()*GB/64L; builder.allocation.initialnumdocs(numDocs); } } diff --git a/config-model/src/main/resources/schema/containercluster.rnc b/config-model/src/main/resources/schema/containercluster.rnc index 2f40a090678..76137fd6263 100644 --- a/config-model/src/main/resources/schema/containercluster.rnc +++ b/config-model/src/main/resources/schema/containercluster.rnc @@ -212,9 +212,11 @@ NodesOfContainerCluster = element nodes { attribute jvm-options { text }? & attribute jvm-gc-options { text }? & attribute preload { text }? & - attribute environment-vars { text }? & attribute allocated-memory { text }? & attribute cpu-socket-affinity { xsd:boolean }? & + element environment-variables { + anyElement + + } ? & ( ( attribute of { xsd:string } & @@ -231,10 +233,6 @@ NodesOfContainerCluster = element nodes { attribute docker-image { xsd:string }? ) | - element environment-variables { - anyElement + - } - | element node { attribute hostalias { xsd:NCName } & attribute cpu-socket { xsd:positiveInteger }? & diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java index 56d53c2cc69..c6e322188c0 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/NodeFlavorTuningTest.java @@ -49,9 +49,9 @@ public class NodeFlavorTuningTest { assertEquals(3, cfg.documentdb().size()); assertEquals(1024, cfg.documentdb(0).allocation().initialnumdocs()); assertEquals("a", cfg.documentdb(0).inputdoctypename()); - assertEquals(644245094L, cfg.documentdb(1).allocation().initialnumdocs()); + assertEquals(402653184, cfg.documentdb(1).allocation().initialnumdocs()); assertEquals("b", cfg.documentdb(1).inputdoctypename()); - assertEquals(644245094L, cfg.documentdb(2).allocation().initialnumdocs()); + assertEquals(402653184, cfg.documentdb(2).allocation().initialnumdocs()); assertEquals("c", cfg.documentdb(2).inputdoctypename()); } diff --git a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java index ec10ffa68a2..50602f27071 100644 --- a/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java +++ b/config-model/src/test/java/com/yahoo/vespa/model/search/test/DocumentDatabaseTestCase.java @@ -118,13 +118,13 @@ public class DocumentDatabaseTestCase { @Test public void requireThatConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency("index", "", 0.2, 0.2); - verifyConcurrency("streaming", "", 0.4, 0.0); - verifyConcurrency("store-only", "", 0.4, 0.0); + verifyConcurrency("index", "", 0.25, 0.25); + verifyConcurrency("streaming", "", 0.5, 0.0); + verifyConcurrency("store-only", "", 0.5, 0.0); } @Test public void requireThatMixedModeConcurrencyIsReflectedCorrectlyForDefault() { - verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 0.4, Arrays.asList(0.2, 0.0)); + verifyConcurrency(Arrays.asList(DocType.create("a", "index"), DocType.create("b", "streaming")), "", 0.5, Arrays.asList(0.25, 0.0)); } @Test public void requireThatMixedModeConcurrencyIsReflected() { diff --git a/config-model/src/test/schema-test-files/services.xml b/config-model/src/test/schema-test-files/services.xml index 880d49fa460..899fd83eca1 100644 --- a/config-model/src/test/schema-test-files/services.xml +++ b/config-model/src/test/schema-test-files/services.xml @@ -210,6 +210,9 @@ <node hostalias="host1"> <server-port id="myServer" port="4090" /> </node> + <environment-variables> + <TEST_VAR>7</TEST_VAR> + </environment-variables> </nodes> </jdisc> diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java index 26fa41c19c7..0e88eeb4e98 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/JRTConfigRequest.java @@ -86,13 +86,6 @@ public interface JRTConfigRequest { long getProtocolVersion(); /** - * Get the wanted generation for this request. - * - * @return a generation that client would like. - */ - long getWantedGeneration(); - - /** * Get the host name of the client that is requesting config. * @return hostname of the client. */ diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java index 584ab1414aa..9a2c4cc63de 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeClientConfigRequest.java @@ -94,11 +94,6 @@ public abstract class SlimeClientConfigRequest implements JRTClientConfigRequest } @Override - public long getWantedGeneration() { - return requestData.getWantedGeneration(); - } - - @Override public Request getRequest() { return request; } diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java index ec6559f4ed7..a9b46f25751 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeRequestData.java @@ -26,7 +26,6 @@ class SlimeRequestData { private static final String REQUEST_CLIENT_CONFIGID = "configId"; private static final String REQUEST_CLIENT_HOSTNAME = "clientHostname"; private static final String REQUEST_CURRENT_GENERATION = "currentGeneration"; - private static final String REQUEST_WANTED_GENERATION = "wantedGeneration"; private static final String REQUEST_CONFIG_MD5 = "configMD5"; private static final String REQUEST_TRACE = "trace"; private static final String REQUEST_TIMEOUT = "timeout"; @@ -73,10 +72,6 @@ class SlimeRequestData { return getRequestField(REQUEST_CLIENT_HOSTNAME).asString(); } - long getWantedGeneration() { - return getRequestField(REQUEST_WANTED_GENERATION).asLong(); - } - long getTimeout() { return getRequestField(REQUEST_TIMEOUT).asLong(); } @@ -110,7 +105,6 @@ class SlimeRequestData { defSchema.serialize(request.setArray(REQUEST_DEF_CONTENT)); request.setString(REQUEST_CONFIG_MD5, configMd5); request.setLong(REQUEST_CURRENT_GENERATION, generation); - request.setLong(REQUEST_WANTED_GENERATION, 0L); // TODO: Remove when latest version in use is 6.328.19 request.setLong(REQUEST_TIMEOUT, timeout); request.setString(REQUEST_COMPRESSION_TYPE, compressionType.name()); vespaVersion.ifPresent(version -> request.setString(REQUEST_VESPA_VERSION, version.toString())); diff --git a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java index 41bf38ef1dc..34d6f90cbcb 100644 --- a/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java +++ b/config/src/main/java/com/yahoo/vespa/config/protocol/SlimeServerConfigRequest.java @@ -79,11 +79,6 @@ abstract class SlimeServerConfigRequest implements JRTServerConfigRequest { } @Override - public long getWantedGeneration() { - return requestData.getWantedGeneration(); - } - - @Override public String getClientHostName() { return requestData.getClientHostName(); } diff --git a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java index cac6cd0fed0..bd7aae3051a 100644 --- a/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java +++ b/config/src/main/java/com/yahoo/vespa/config/util/ConfigUtils.java @@ -15,6 +15,7 @@ import java.io.*; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.text.DecimalFormat; +import java.text.DecimalFormatSymbols; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -28,9 +29,9 @@ public class ConfigUtils { private static final Pattern doublePattern = Pattern.compile(".*double.*range.*"); private static final Pattern spaceBeforeCommaPatter = Pattern.compile("\\s,"); public static final String intFormattedMax = new DecimalFormat("#.#").format(0x7fffffff); - public static final String intFormattedMin = new DecimalFormat("#.#").format(-0x80000000); + public static final String intFormattedMin = new DecimalFormat("#.#", new DecimalFormatSymbols(Locale.ENGLISH)).format(-0x80000000); public static final String doubleFormattedMax = new DecimalFormat("#.#").format(1e308); - public static final String doubleFormattedMin = new DecimalFormat("#.#").format(-1e308); + public static final String doubleFormattedMin = new DecimalFormat("#.#", new DecimalFormatSymbols(Locale.ENGLISH)).format(-1e308); /** * Computes Md5 hash of a list of strings. The only change to input lines before diff --git a/config/src/tests/frt/frt.cpp b/config/src/tests/frt/frt.cpp index b616861c799..090991b7fe4 100644 --- a/config/src/tests/frt/frt.cpp +++ b/config/src/tests/frt/frt.cpp @@ -270,12 +270,11 @@ TEST("require that v3 request is correctly initialized") { ConfigKey key = ConfigKey::create<MyConfig>("foobi"); vespalib::string md5 = "mymd5"; int64_t currentGeneration = 3; - int64_t wantedGeneration = 4; vespalib::string hostName = "myhost"; int64_t timeout = 3000; Trace traceIn(3); traceIn.trace(2, "Hei"); - FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, wantedGeneration, hostName, + FRTConfigRequestV3 v3req(&conn, key, md5, currentGeneration, hostName, timeout, traceIn, VespaVersion::fromString("1.2.3"), CompressionType::LZ4); ASSERT_TRUE(v3req.verifyState(ConfigState(md5, 3, false))); ASSERT_FALSE(v3req.verifyState(ConfigState(md5, 2, false))); @@ -298,7 +297,6 @@ TEST("require that v3 request is correctly initialized") { EXPECT_EQUAL(key.getConfigId(), root[REQUEST_CLIENT_CONFIGID].asString().make_string()); EXPECT_EQUAL(hostName, root[REQUEST_CLIENT_HOSTNAME].asString().make_string()); EXPECT_EQUAL(currentGeneration, root[REQUEST_CURRENT_GENERATION].asLong()); - EXPECT_EQUAL(wantedGeneration, root[REQUEST_WANTED_GENERATION].asLong()); EXPECT_EQUAL(md5, root[REQUEST_CONFIG_MD5].asString().make_string()); EXPECT_EQUAL(timeout, root[REQUEST_TIMEOUT].asLong()); EXPECT_EQUAL("LZ4", root[REQUEST_COMPRESSION_TYPE].asString().make_string()); diff --git a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp index 1f1ddb196b0..4b5d1621c62 100644 --- a/config/src/vespa/config/frt/frtconfigrequestfactory.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestfactory.cpp @@ -26,7 +26,7 @@ FRTConfigRequest::UP FRTConfigRequestFactory::createConfigRequest(const ConfigKey & key, Connection * connection, const ConfigState & state, int64_t serverTimeout) const { - return make_unique<FRTConfigRequestV3>(connection, key, state.md5, state.generation, 0u, _hostName, + return make_unique<FRTConfigRequestV3>(connection, key, state.md5, state.generation, _hostName, serverTimeout, Trace(_traceLevel), _vespaVersion, _compressionType); } diff --git a/config/src/vespa/config/frt/frtconfigrequestv3.cpp b/config/src/vespa/config/frt/frtconfigrequestv3.cpp index 413c74c796d..e52ce5d1114 100644 --- a/config/src/vespa/config/frt/frtconfigrequestv3.cpp +++ b/config/src/vespa/config/frt/frtconfigrequestv3.cpp @@ -13,13 +13,12 @@ FRTConfigRequestV3::FRTConfigRequestV3(Connection * connection, const ConfigKey & key, const vespalib::string & configMd5, int64_t currentGeneration, - int64_t wantedGeneration, const vespalib::string & hostName, int64_t serverTimeout, const Trace & trace, const VespaVersion & vespaVersion, const CompressionType & compressionType) - : SlimeConfigRequest(connection, key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, vespaVersion, 3, compressionType, "config.v3.getConfig") + : SlimeConfigRequest(connection, key, configMd5, currentGeneration, hostName, serverTimeout, trace, vespaVersion, 3, compressionType, "config.v3.getConfig") { } diff --git a/config/src/vespa/config/frt/frtconfigrequestv3.h b/config/src/vespa/config/frt/frtconfigrequestv3.h index d8c767bc936..3b671e52c08 100644 --- a/config/src/vespa/config/frt/frtconfigrequestv3.h +++ b/config/src/vespa/config/frt/frtconfigrequestv3.h @@ -19,7 +19,6 @@ public: const ConfigKey & key, const vespalib::string & configMd5, int64_t currentGeneration, - int64_t wantedGeneration, const vespalib::string & hostName, int64_t serverTimeout, const Trace & trace, diff --git a/config/src/vespa/config/frt/protocol.cpp b/config/src/vespa/config/frt/protocol.cpp index 9f8ea9a06fa..4ad55726863 100644 --- a/config/src/vespa/config/frt/protocol.cpp +++ b/config/src/vespa/config/frt/protocol.cpp @@ -25,7 +25,6 @@ const Memory REQUEST_CLIENT_CONFIGID = "configId"; const Memory REQUEST_CLIENT_HOSTNAME = "clientHostname"; const Memory REQUEST_CONFIG_MD5 = "configMD5"; const Memory REQUEST_CURRENT_GENERATION = "currentGeneration"; -const Memory REQUEST_WANTED_GENERATION = "wantedGeneration"; const Memory REQUEST_TIMEOUT = "timeout"; const Memory REQUEST_TRACE = "trace"; const Memory REQUEST_VESPA_VERSION = "vespaVersion"; diff --git a/config/src/vespa/config/frt/protocol.h b/config/src/vespa/config/frt/protocol.h index 2c96d355a1d..176f2c34026 100644 --- a/config/src/vespa/config/frt/protocol.h +++ b/config/src/vespa/config/frt/protocol.h @@ -38,7 +38,6 @@ extern const vespalib::Memory REQUEST_CLIENT_CONFIGID; extern const vespalib::Memory REQUEST_CLIENT_HOSTNAME; extern const vespalib::Memory REQUEST_CONFIG_MD5; extern const vespalib::Memory REQUEST_CURRENT_GENERATION; -extern const vespalib::Memory REQUEST_WANTED_GENERATION; extern const vespalib::Memory REQUEST_TIMEOUT; extern const vespalib::Memory REQUEST_TRACE; extern const vespalib::Memory REQUEST_VESPA_VERSION; diff --git a/config/src/vespa/config/frt/slimeconfigrequest.cpp b/config/src/vespa/config/frt/slimeconfigrequest.cpp index 27ac39ae56a..07626c1e274 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.cpp +++ b/config/src/vespa/config/frt/slimeconfigrequest.cpp @@ -23,7 +23,6 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, const ConfigKey & key, const vespalib::string & configMd5, int64_t currentGeneration, - int64_t wantedGeneration, const vespalib::string & hostName, int64_t serverTimeout, const Trace & trace, @@ -34,7 +33,7 @@ SlimeConfigRequest::SlimeConfigRequest(Connection * connection, : FRTConfigRequest(connection, key), _data() { - populateSlimeRequest(key, configMd5, currentGeneration, wantedGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); + populateSlimeRequest(key, configMd5, currentGeneration, hostName, serverTimeout, trace, vespaVersion, protocolVersion, compressionType); _request->SetMethodName(methodName.c_str()); _parameters.AddString(createJsonFromSlime(_data).c_str()); } @@ -50,7 +49,6 @@ void SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, const vespalib::string & configMd5, int64_t currentGeneration, - int64_t wantedGeneration, const vespalib::string & hostName, int64_t serverTimeout, const Trace & trace, @@ -69,7 +67,6 @@ SlimeConfigRequest::populateSlimeRequest(const ConfigKey & key, root.setString(REQUEST_CLIENT_HOSTNAME, Memory(hostName)); root.setString(REQUEST_CONFIG_MD5, Memory(configMd5)); root.setLong(REQUEST_CURRENT_GENERATION, currentGeneration); - root.setLong(REQUEST_WANTED_GENERATION, wantedGeneration); root.setLong(REQUEST_TIMEOUT, serverTimeout); trace.serialize(root.setObject(REQUEST_TRACE)); root.setString(REQUEST_COMPRESSION_TYPE, Memory(compressionTypeToString(compressionType))); diff --git a/config/src/vespa/config/frt/slimeconfigrequest.h b/config/src/vespa/config/frt/slimeconfigrequest.h index 625d70c094b..60c44b6c97f 100644 --- a/config/src/vespa/config/frt/slimeconfigrequest.h +++ b/config/src/vespa/config/frt/slimeconfigrequest.h @@ -21,7 +21,6 @@ public: const ConfigKey & key, const vespalib::string & configMd5, int64_t currentGeneration, - int64_t wantedGeneration, const vespalib::string & hostName, int64_t serverTimeout, const Trace & trace, @@ -36,7 +35,6 @@ private: void populateSlimeRequest(const ConfigKey & key, const vespalib::string & configMd5, int64_t currentGeneration, - int64_t wantedGeneration, const vespalib::string & hostName, int64_t serverTimeout, const Trace & trace, diff --git a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java index 4e1c79248a9..ce3cbddb6e1 100644 --- a/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java +++ b/controller-server/src/main/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAlias.java @@ -78,8 +78,7 @@ public class LoadBalancerAlias { ignorePartIfDefault(applicationId.instance().value()), applicationId.application().value(), applicationId.tenant().value(), - zoneId.value(), - "vespa.oath.cloud" + zoneId.value() + "." + "vespa.oath.cloud" ); return parts.stream() .filter(s -> !Strings.isNullOrEmpty((s))) diff --git a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java index 965a639a68c..c0e58078f79 100644 --- a/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java +++ b/controller-server/src/test/java/com/yahoo/vespa/hosted/controller/application/LoadBalancerAliasTest.java @@ -18,12 +18,12 @@ public class LoadBalancerAliasTest { public void test_endpoint_names() { ZoneId zoneId = ZoneId.from("prod", "us-north-1"); ApplicationId withInstanceName = ApplicationId.from("tenant", "application", "instance"); - testAlias("instance--application--tenant--prod.us-north-1--vespa.oath.cloud", "default", withInstanceName, zoneId); - testAlias("cluster--instance--application--tenant--prod.us-north-1--vespa.oath.cloud", "cluster", withInstanceName, zoneId); + testAlias("instance--application--tenant--prod.us-north-1.vespa.oath.cloud", "default", withInstanceName, zoneId); + testAlias("cluster--instance--application--tenant--prod.us-north-1.vespa.oath.cloud", "cluster", withInstanceName, zoneId); ApplicationId withDefaultInstance = ApplicationId.from("tenant", "application", "default"); - testAlias("application--tenant--prod.us-north-1--vespa.oath.cloud", "default", withDefaultInstance, zoneId); - testAlias("cluster--application--tenant--prod.us-north-1--vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); + testAlias("application--tenant--prod.us-north-1.vespa.oath.cloud", "default", withDefaultInstance, zoneId); + testAlias("cluster--application--tenant--prod.us-north-1.vespa.oath.cloud", "cluster", withDefaultInstance, zoneId); } private void testAlias(String expected, String clusterName, ApplicationId applicationId, ZoneId zoneId) { diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp index 658ebe818eb..340619f09bd 100644 --- a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -22,11 +22,10 @@ struct Fixture { builder.indexing.threads = baseLineIndexingThreads; builder.indexing.tasklimit = 500; builder.indexing.semiunboundtasklimit = 50000; - builder.feeding.concurrency = 0.5; return builder; } ThreadingServiceConfig make(uint32_t cpuCores) { - return ThreadingServiceConfig::make(cfg, HwInfo::Cpu(cpuCores)); + return ThreadingServiceConfig::make(cfg, 0.5, HwInfo::Cpu(cpuCores)); } void assertIndexingThreads(uint32_t expIndexingThreads, uint32_t cpuCores) { EXPECT_EQUAL(expIndexingThreads, make(cpuCores).indexingThreads()); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 6a1d053745f..352de412f79 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -68,20 +68,20 @@ namespace proton { namespace { constexpr uint32_t indexing_thread_stack_size = 128 * 1024; +using Allocation = ProtonConfig::Documentdb::Allocation; GrowStrategy -makeGrowStrategy(uint32_t docsInitialCapacity, const ProtonConfig::Grow &growCfg) +makeGrowStrategy(uint32_t docsInitialCapacity, const Allocation &allocCfg) { - return GrowStrategy(docsInitialCapacity, growCfg.factor, growCfg.add, growCfg.multivalueallocfactor); + return GrowStrategy(docsInitialCapacity, allocCfg.growfactor, allocCfg.growbias, allocCfg.multivaluegrowfactor); } DocumentSubDBCollection::Config -makeSubDBConfig(const ProtonConfig & protonCfg) { - const ProtonConfig::Grow & growCfg = protonCfg.grow; - const ProtonConfig::Distribution & distCfg = protonCfg.distribution; - GrowStrategy searchableGrowth = makeGrowStrategy(growCfg.initial * distCfg.searchablecopies, growCfg); - GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024l, growCfg.initial/100), growCfg); - GrowStrategy notReadyGrowth = makeGrowStrategy(growCfg.initial * (distCfg.redundancy - distCfg.searchablecopies), growCfg); - return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, growCfg.numdocs, protonCfg.numsearcherthreads); +makeSubDBConfig(const ProtonConfig::Distribution & distCfg, const Allocation & allocCfg, size_t numSearcherThreads) { + size_t initialNumDocs(allocCfg.initialnumdocs); + GrowStrategy searchableGrowth = makeGrowStrategy(initialNumDocs * distCfg.searchablecopies, allocCfg); + GrowStrategy removedGrowth = makeGrowStrategy(std::max(1024ul, initialNumDocs/100), allocCfg); + GrowStrategy notReadyGrowth = makeGrowStrategy(initialNumDocs * (distCfg.redundancy - distCfg.searchablecopies), allocCfg); + return DocumentSubDBCollection::Config(searchableGrowth, notReadyGrowth, removedGrowth, allocCfg.amortizecount, numSearcherThreads); } index::IndexConfig @@ -89,6 +89,18 @@ makeIndexConfig(const ProtonConfig::Index & cfg) { return index::IndexConfig(WarmupConfig(cfg.warmup.time, cfg.warmup.unpack), cfg.maxflushed, cfg.cache.size); } +ProtonConfig::Documentdb _G_defaultProtonDocumentDBConfig; + +const ProtonConfig::Documentdb * +findDocumentDB(const ProtonConfig::DocumentdbVector & documentDBs, const vespalib::string & docType) { + for (const auto & dbCfg : documentDBs) { + if (dbCfg.inputdoctypename == docType) { + return & dbCfg; + } + } + return &_G_defaultProtonDocumentDBConfig; +} + } template <typename FunctionType> @@ -124,7 +136,10 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _bucketSpace(bucketSpace), _baseDir(baseDir + "/" + _docTypeName.toString()), // Only one thread per executor, or performDropFeedView() will fail. - _writeServiceConfig(ThreadingServiceConfig::make(protonCfg, hwInfo.cpu())), + _writeServiceConfig( + ThreadingServiceConfig::make(protonCfg, + findDocumentDB(protonCfg.documentdb, docTypeName.getName())->feeding.concurrency, + hwInfo.cpu())), _writeService(_writeServiceConfig.indexingThreads(), indexing_thread_stack_size, _writeServiceConfig.defaultTaskLimit()), @@ -142,7 +157,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _bucketHandler(_writeService.master()), _indexCfg(makeIndexConfig(protonCfg.index)), _config_store(std::move(config_store)), - _sessionManager(new matching::SessionManager(protonCfg.grouping.sessionmanager.maxentries)), + _sessionManager(std::make_shared<matching::SessionManager>(protonCfg.grouping.sessionmanager.maxentries)), _metricsWireService(metricsWireService), _metricsHook(*this, _docTypeName.getName(), protonCfg.numthreadspersearch), _feedView(), @@ -153,9 +168,12 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _dmUsageForwarder(_writeService.master()), _writeFilter(), _feedHandler(_writeService, tlsSpec, docTypeName, _state, *this, _writeFilter, *this, tlsDirectWriter), - _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, - sharedExecutor, fileHeaderContext, metricsWireService, getMetrics(), - queryLimiter, clock, _configMutex, _baseDir, makeSubDBConfig(protonCfg), hwInfo), + _subDBs(*this, *this, _feedHandler, _docTypeName, _writeService, warmupExecutor, sharedExecutor, fileHeaderContext, + metricsWireService, getMetrics(), queryLimiter, clock, _configMutex, _baseDir, + makeSubDBConfig(protonCfg.distribution, + findDocumentDB(protonCfg.documentdb, docTypeName.getName())->allocation, + protonCfg.numsearcherthreads), + hwInfo), _maintenanceController(_writeService.master(), sharedExecutor, _docTypeName), _visibility(_feedHandler, _writeService, _feedView), _lidSpaceCompactionHandlers(), @@ -165,8 +183,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, { assert(configSnapshot); - LOG(debug, "DocumentDB(%s): Creating database in directory '%s'", - _docTypeName.toString().c_str(), _baseDir.c_str()); + LOG(debug, "DocumentDB(%s): Creating database in directory '%s'", _docTypeName.toString().c_str(), _baseDir.c_str()); _feedHandler.init(_config_store->getOldestSerialNum()); _feedHandler.setBucketDBHandler(&_subDBs.getBucketDBHandler()); diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp index cd1608feadb..8f1c3560e9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -20,21 +20,20 @@ ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_, namespace { uint32_t -calculateIndexingThreads(const ProtonConfig &cfg, const HwInfo::Cpu &cpuInfo) +calculateIndexingThreads(uint32_t cfgIndexingThreads, double concurrency, const HwInfo::Cpu &cpuInfo) { - double scaledCores = cpuInfo.cores() * cfg.feeding.concurrency; - uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), (uint32_t)cfg.indexing.threads); + double scaledCores = cpuInfo.cores() * concurrency; + uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), cfgIndexingThreads); return std::max(indexingThreads, 1u); } } ThreadingServiceConfig -ThreadingServiceConfig::make(const ProtonConfig &cfg, const HwInfo::Cpu &cpuInfo) +ThreadingServiceConfig::make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo) { - uint32_t indexingThreads = calculateIndexingThreads(cfg, cpuInfo); - return ThreadingServiceConfig(indexingThreads, - cfg.indexing.tasklimit, + uint32_t indexingThreads = calculateIndexingThreads(cfg.indexing.threads, concurrency, cpuInfo); + return ThreadingServiceConfig(indexingThreads, cfg.indexing.tasklimit, (cfg.indexing.semiunboundtasklimit / indexingThreads)); } diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h index 03c0c4d1842..be39f516598 100644 --- a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h @@ -20,13 +20,10 @@ private: uint32_t _semiUnboundTaskLimit; private: - ThreadingServiceConfig(uint32_t indexingThreads_, - uint32_t defaultTaskLimit_, - uint32_t semiUnboundTaskLimit_); + ThreadingServiceConfig(uint32_t indexingThreads_, uint32_t defaultTaskLimit_, uint32_t semiUnboundTaskLimit_); public: - static ThreadingServiceConfig make(const ProtonConfig &cfg, - const HwInfo::Cpu &cpuInfo); + static ThreadingServiceConfig make(const ProtonConfig &cfg, double concurrency, const HwInfo::Cpu &cpuInfo); uint32_t indexingThreads() const { return _indexingThreads; } uint32_t defaultTaskLimit() const { return _defaultTaskLimit; } |