diff options
6 files changed, 155 insertions, 0 deletions
diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 0dce35f7eb3..2944b753dca 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -97,6 +97,7 @@ vespa_define_module( src/tests/proton/documentdb/maintenancecontroller src/tests/proton/documentdb/move_operation_limiter src/tests/proton/documentdb/storeonlyfeedview + src/tests/proton/documentdb/threading_service_config src/tests/proton/documentmetastore src/tests/proton/documentmetastore/lidreusedelayer src/tests/proton/feed_and_search diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/CMakeLists.txt b/searchcore/src/tests/proton/documentdb/threading_service_config/CMakeLists.txt new file mode 100644 index 00000000000..214b5c9b86d --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/CMakeLists.txt @@ -0,0 +1,8 @@ +# Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_add_executable(searchcore_threading_service_config_test_app TEST + SOURCES + threading_service_config_test.cpp + DEPENDS + searchcore_server +) +vespa_add_test(NAME searchcore_threading_service_config_test_app COMMAND searchcore_threading_service_config_test_app) diff --git a/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp new file mode 100644 index 00000000000..658ebe818eb --- /dev/null +++ b/searchcore/src/tests/proton/documentdb/threading_service_config/threading_service_config_test.cpp @@ -0,0 +1,66 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/log/log.h> +LOG_SETUP("threading_service_config_test"); + +#include <vespa/searchcore/config/config-proton.h> +#include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchcore/proton/server/threading_service_config.h> +#include <vespa/vespalib/testkit/testapp.h> + +using namespace proton; +using ProtonConfig = vespa::config::search::core::ProtonConfig; +using ProtonConfigBuilder = vespa::config::search::core::ProtonConfigBuilder; + +struct Fixture { + ProtonConfig cfg; + Fixture(uint32_t baseLineIndexingThreads = 2) + : cfg(makeConfig(baseLineIndexingThreads)) + { + } + ProtonConfig makeConfig(uint32_t baseLineIndexingThreads) { + ProtonConfigBuilder builder; + builder.indexing.threads = baseLineIndexingThreads; + builder.indexing.tasklimit = 500; + builder.indexing.semiunboundtasklimit = 50000; + builder.feeding.concurrency = 0.5; + return builder; + } + ThreadingServiceConfig make(uint32_t cpuCores) { + return ThreadingServiceConfig::make(cfg, HwInfo::Cpu(cpuCores)); + } + void assertIndexingThreads(uint32_t expIndexingThreads, uint32_t cpuCores) { + EXPECT_EQUAL(expIndexingThreads, make(cpuCores).indexingThreads()); + } +}; + +TEST_F("require that indexing threads are set based on cpu cores and feeding concurrency", Fixture) +{ + TEST_DO(f.assertIndexingThreads(2, 1)); + TEST_DO(f.assertIndexingThreads(2, 4)); + TEST_DO(f.assertIndexingThreads(2, 8)); + TEST_DO(f.assertIndexingThreads(2, 12)); + TEST_DO(f.assertIndexingThreads(3, 13)); + TEST_DO(f.assertIndexingThreads(3, 18)); + TEST_DO(f.assertIndexingThreads(4, 19)); + TEST_DO(f.assertIndexingThreads(4, 24)); +} + +TEST_F("require that indexing threads is always >= 1", Fixture(0)) +{ + TEST_DO(f.assertIndexingThreads(1, 0)); +} + +TEST_F("require that default task limit is set", Fixture) +{ + EXPECT_EQUAL(500u, f.make(24).defaultTaskLimit()); +} + +TEST_F("require that semiunbound task limit is scaled based on indexing threads", Fixture) +{ + EXPECT_EQUAL(12500u, f.make(24).semiUnboundTaskLimit()); +} + +TEST_MAIN() +{ + TEST_RUN_ALL(); +} diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 1716441c30b..5bb512c12ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -93,6 +93,7 @@ vespa_add_library(searchcore_server STATIC storeonlydocsubdb.cpp storeonlyfeedview.cpp summaryadapter.cpp + threading_service_config.cpp tlcproxy.cpp tlssyncer.cpp transactionlogmanager.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp new file mode 100644 index 00000000000..6422df9cbd2 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.cpp @@ -0,0 +1,43 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "threading_service_config.h" +#include <vespa/searchcore/proton/common/hw_info.h> +#include <cmath> + +namespace proton { + +using ProtonConfig = ThreadingServiceConfig::ProtonConfig; + +ThreadingServiceConfig::ThreadingServiceConfig(uint32_t indexingThreads_, + uint32_t defaultTaskLimit_, + uint32_t semiUnboundTaskLimit_) + : _indexingThreads(indexingThreads_), + _defaultTaskLimit(defaultTaskLimit_), + _semiUnboundTaskLimit(semiUnboundTaskLimit_) +{ +} + +namespace { + +uint32_t +calculateIndexingThreads(const ProtonConfig &cfg, + const HwInfo::Cpu &cpuInfo) +{ + double scaledCores = cpuInfo.cores() * cfg.feeding.concurrency; + uint32_t indexingThreads = std::max((uint32_t)std::ceil(scaledCores / 3), (uint32_t)cfg.indexing.threads); + return std::max(indexingThreads, 1u); +} + +} + +ThreadingServiceConfig +ThreadingServiceConfig::make(const ProtonConfig &cfg, + const HwInfo::Cpu &cpuInfo) +{ + uint32_t indexingThreads = calculateIndexingThreads(cfg, cpuInfo); + return ThreadingServiceConfig(indexingThreads, + cfg.indexing.tasklimit, + (cfg.indexing.semiunboundtasklimit / indexingThreads)); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h new file mode 100644 index 00000000000..67ab4171e80 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/threading_service_config.h @@ -0,0 +1,36 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +#include <vespa/searchcore/config/config-proton.h> +#include <vespa/searchcore/proton/common/hw_info.h> +#include <cstdint> + +namespace proton { + +/** + * Config for the threading service used by a documentdb. + */ +class ThreadingServiceConfig { +public: + using ProtonConfig = vespa::config::search::core::ProtonConfig; + +private: + uint32_t _indexingThreads; + uint32_t _defaultTaskLimit; + uint32_t _semiUnboundTaskLimit; + +private: + ThreadingServiceConfig(uint32_t indexingThreads_, + uint32_t defaultTaskLimit_, + uint32_t semiUnboundTaskLimit_); + +public: + static ThreadingServiceConfig make(const ProtonConfig &cfg, + const HwInfo::Cpu &cpuInfo); + + uint32_t indexingThreads() const { return _indexingThreads; } + uint32_t defaultTaskLimit() const { return _defaultTaskLimit; } + uint32_t semiUnboundTaskLimit() const { return _semiUnboundTaskLimit; } +}; + +} |