diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2023-04-25 21:11:04 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2023-04-25 21:11:04 +0000 |
commit | a31d90fe3960db716d4f7406d024c9b4eec9a173 (patch) | |
tree | f3a9c038f1212ed03776bf7413c0d3a20a280e70 | |
parent | 51f5ab6eb26d901e5018094338db7cd59ded9106 (diff) |
Cap the initialize thread pool at number of cores
4 files changed, 30 insertions, 8 deletions
diff --git a/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp b/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp index 7aad69076ca..a34af63371c 100644 --- a/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp +++ b/searchcore/src/tests/proton/server/initialize_threads_calculator/initialize_threads_calculator_test.cpp @@ -16,7 +16,9 @@ public: void expect_successful_init(uint32_t exp_threads) { - InitializeThreadsCalculator i("tmp", 9); + constexpr uint32_t cfg_threads = 9; + HwInfo hw_info(HwInfo::Disk(1,false, false), HwInfo::Memory(1), HwInfo::Cpu(cfg_threads)); + InitializeThreadsCalculator i(hw_info, "tmp", cfg_threads); EXPECT_EQ(exp_threads, i.num_threads()); EXPECT_TRUE(i.threads().get() != nullptr); EXPECT_EQ(exp_threads, dynamic_cast<const ThreadStackExecutor&>(*i.threads()).getNumThreads()); @@ -27,7 +29,8 @@ expect_successful_init(uint32_t exp_threads) void expect_aborted_init(uint32_t exp_threads, uint32_t cfg_threads = 9) { - InitializeThreadsCalculator i("tmp", cfg_threads); + HwInfo hw_info(HwInfo::Disk(1,false, false), HwInfo::Memory(1), HwInfo::Cpu(cfg_threads)); + InitializeThreadsCalculator i(hw_info, "tmp", cfg_threads); EXPECT_EQ(exp_threads, i.num_threads()); EXPECT_TRUE(i.threads().get() != nullptr); EXPECT_EQ(exp_threads, dynamic_cast<const ThreadStackExecutor&>(*i.threads()).getNumThreads()); @@ -53,7 +56,8 @@ TEST_F(InitializeThreadsCalculatorTest, initialize_threads_cut_in_half_when_init TEST_F(InitializeThreadsCalculatorTest, zero_initialize_threads_is_special) { { - InitializeThreadsCalculator i("tmp", 0); + HwInfo hw_info(HwInfo::Disk(1,false, false), HwInfo::Memory(1), HwInfo::Cpu(10)); + InitializeThreadsCalculator i(hw_info, "tmp", 0); EXPECT_EQ(0, i.num_threads()); EXPECT_TRUE(i.threads().get() == nullptr); } @@ -61,4 +65,21 @@ TEST_F(InitializeThreadsCalculatorTest, zero_initialize_threads_is_special) expect_aborted_init(1, 0); } +void +expect_lower(uint32_t cores, uint32_t configured) { + HwInfo hw_info(HwInfo::Disk(1,false, false), HwInfo::Memory(1), HwInfo::Cpu(cores)); + InitializeThreadsCalculator i(hw_info, "tmp", configured); + EXPECT_EQ(std::min(cores, configured), i.num_threads()); + i.init_done(); +} + +TEST_F(InitializeThreadsCalculatorTest, lower_of_wanted_and_cores) +{ + expect_lower(1, 7); + expect_lower(6, 7); + expect_lower(7, 7); + expect_lower(7, 6); + expect_lower(7, 1); +} + GTEST_MAIN_RUN_ALL_TESTS() diff --git a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp index 17ddabebb65..e3a3a978e68 100644 --- a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.cpp @@ -39,10 +39,11 @@ const vespalib::string file_name = "initialize-threads.txt"; namespace proton { -InitializeThreadsCalculator::InitializeThreadsCalculator(const vespalib::string& base_dir, +InitializeThreadsCalculator::InitializeThreadsCalculator(const HwInfo & hw_info, + const vespalib::string& base_dir, uint32_t configured_num_threads) : _path(base_dir + "/" + file_name), - _num_threads(configured_num_threads), + _num_threads(std::min(hw_info.cpu().cores(), configured_num_threads)), _threads() { if (std::filesystem::exists(_path)) { diff --git a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h index 509db24ebf3..bfa6102066a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h +++ b/searchcore/src/vespa/searchcore/proton/server/initialize_threads_calculator.h @@ -2,6 +2,7 @@ #pragma once +#include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/threadexecutor.h> #include <filesystem> @@ -25,8 +26,7 @@ namespace proton { InitializeThreads _threads; public: - InitializeThreadsCalculator(const vespalib::string& base_dir, - uint32_t configured_num_threads); + InitializeThreadsCalculator(const HwInfo & hwInfo, const vespalib::string& base_dir, uint32_t configured_num_threads); ~InitializeThreadsCalculator(); uint32_t num_threads() const { return _num_threads; } InitializeThreads threads() const { return _threads; } diff --git a/searchcore/src/vespa/searchcore/proton/server/proton.cpp b/searchcore/src/vespa/searchcore/proton/server/proton.cpp index 8ec904760cf..0a95b836544 100644 --- a/searchcore/src/vespa/searchcore/proton/server/proton.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/proton.cpp @@ -352,7 +352,7 @@ Proton::init(const BootstrapConfig::SP & configSnapshot) vespalib::string fileConfigId; _compile_cache_executor_binding = vespalib::eval::CompileCache::bind(_shared_service->shared_raw()); - InitializeThreadsCalculator calc(protonConfig.basedir, protonConfig.initialize.threads); + InitializeThreadsCalculator calc(hwInfo, protonConfig.basedir, protonConfig.initialize.threads); LOG(info, "Start initializing components: threads=%u, configured=%u", calc.num_threads(), protonConfig.initialize.threads); _initDocumentDbsInSequence = (calc.num_threads() == 1); |