diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2022-12-19 16:06:16 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2022-12-20 05:10:52 +0000 |
commit | a00e560b0d8267e9b376e5c0e6a2139a7be63281 (patch) | |
tree | ba207915b8cac9fb92493cc103821f6cce9db31b /searchlib | |
parent | 05b58ac83b06b00ae97ecafad101e44d4dd76aee (diff) |
Remove stacksize from the thread pools and thread executors.
Diffstat (limited to 'searchlib')
20 files changed, 40 insertions, 38 deletions
diff --git a/searchlib/src/apps/docstore/benchmarkdatastore.cpp b/searchlib/src/apps/docstore/benchmarkdatastore.cpp index 9c2a40d6e98..cf2e7f7356d 100644 --- a/searchlib/src/apps/docstore/benchmarkdatastore.cpp +++ b/searchlib/src/apps/docstore/benchmarkdatastore.cpp @@ -97,12 +97,12 @@ BenchmarkDataStoreApp::benchmark(const vespalib::string & dir, size_t numReads, tuning._randRead.setWantMemoryMap(); } search::index::DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); transactionlog::NoSyncProxy noTlSyncer; LogDataStore store(executor, dir, config, growStrategy, tuning, fileHeaderContext, noTlSyncer, NULL, true); - vespalib::ThreadStackExecutor bmPool(numThreads, 128_Ki); + vespalib::ThreadStackExecutor bmPool(numThreads); LOG(info, "Start read benchmark with %lu threads doing %lu reads in chunks of %lu reads. Totally %lu objects", numThreads, numReads, perChunk, numThreads * numReads * perChunk); for (size_t i(0); i < numThreads; i++) { bmPool.execute(vespalib::makeLambdaTask([&]() { read(numReads, perChunk, static_cast<const IDataStore *>(&store)); })); diff --git a/searchlib/src/apps/docstore/documentstoreinspect.cpp b/searchlib/src/apps/docstore/documentstoreinspect.cpp index 34afa0216ed..f19a128535f 100644 --- a/searchlib/src/apps/docstore/documentstoreinspect.cpp +++ b/searchlib/src/apps/docstore/documentstoreinspect.cpp @@ -108,7 +108,7 @@ DocumentStoreInspectApp::verify(const vespalib::string & dir) GrowStrategy growStrategy; TuneFileSummary tuning; search::index::DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); transactionlog::NoSyncProxy noTlSyncer; LogDataStore store(executor, dir, config, growStrategy, tuning, diff --git a/searchlib/src/apps/docstore/verifylogdatastore.cpp b/searchlib/src/apps/docstore/verifylogdatastore.cpp index 3b673c172c9..f803f7aee9f 100644 --- a/searchlib/src/apps/docstore/verifylogdatastore.cpp +++ b/searchlib/src/apps/docstore/verifylogdatastore.cpp @@ -51,7 +51,7 @@ VerifyLogDataStoreApp::verify(const vespalib::string & dir) GrowStrategy growStrategy; TuneFileSummary tuning; search::index::DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); transactionlog::NoSyncProxy noTlSyncer; try { diff --git a/searchlib/src/apps/tests/biglogtest.cpp b/searchlib/src/apps/tests/biglogtest.cpp index bd8991edc4b..154e94d0c66 100644 --- a/searchlib/src/apps/tests/biglogtest.cpp +++ b/searchlib/src/apps/tests/biglogtest.cpp @@ -4,7 +4,6 @@ #include <vespa/searchlib/docstore/logdatastore.h> #include <vespa/searchlib/index/dummyfileheadercontext.h> #include <vespa/searchlib/transactionlog/nosyncproxy.h> -#include <vespa/vespalib/util/exceptions.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/threadstackexecutor.h> #include <vespa/vespalib/data/databuffer.h> @@ -136,7 +135,7 @@ factory<LogDataStore>::factory(std::string dir) : DioTune(), _fileHeaderContext(), _config(), - _executor(1, 128_Ki), + _executor(1), _noTlSyncer(), _datastore(_executor, dir, _config, GrowStrategy(), tuning, _fileHeaderContext, _noTlSyncer, NULL) {} diff --git a/searchlib/src/apps/tests/memoryindexstress_test.cpp b/searchlib/src/apps/tests/memoryindexstress_test.cpp index 526b4cad9a6..fb743ad93ac 100644 --- a/searchlib/src/apps/tests/memoryindexstress_test.cpp +++ b/searchlib/src/apps/tests/memoryindexstress_test.cpp @@ -8,7 +8,6 @@ #include <vespa/searchlib/memoryindex/memory_index.h> #include <vespa/searchlib/query/tree/simplequery.h> #include <vespa/searchlib/queryeval/fake_requestcontext.h> -#include <vespa/searchlib/queryeval/fake_search.h> #include <vespa/searchlib/queryeval/fake_searchable.h> #include <vespa/searchlib/queryeval/searchiterator.h> #include <vespa/searchlib/queryeval/blueprint.h> @@ -265,13 +264,13 @@ VESPA_THREAD_STACK_TAG(push_executor) Fixture::Fixture(uint32_t readThreads) : schema(makeSchema()), repo(makeDocTypeRepoConfig()), - _executor(1, 128_Ki), + _executor(1), _invertThreads(vespalib::SequencedTaskExecutor::create(invert_executor, 2)), _pushThreads(vespalib::SequencedTaskExecutor::create(push_executor, 2)), index(schema, MockFieldLengthInspector(), *_invertThreads, *_pushThreads), _readThreads(readThreads), - _writer(1, 128_Ki), - _readers(readThreads, 128_Ki), + _writer(1), + _readers(readThreads), _rnd(), _keyLimit(1000000), _readSeed(50), diff --git a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp index cb6c66915f1..39e5d713d08 100644 --- a/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp +++ b/searchlib/src/tests/attribute/benchmark/attributebenchmark.cpp @@ -569,7 +569,7 @@ AttributeBenchmark::main(int argc, char **argv) dc._attribute = vespalib::string(argv[optind]); - _threadPool = new FastOS_ThreadPool(256000); + _threadPool = new FastOS_ThreadPool(); std::cout << "<attribute-benchmark>" << std::endl; init(dc); diff --git a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp index b145192eb3a..584335803a8 100644 --- a/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp +++ b/searchlib/src/tests/attribute/tensorattribute/tensorattribute_test.cpp @@ -582,7 +582,7 @@ Fixture::Fixture(const vespalib::string &typeSpec, FixtureTraits traits) _index_factory(), _tensorAttr(), _attr(), - _executor(1, 0x10000), + _executor(1), _denseTensors(false), _traits(traits), _mmap_allocator_base_dir("mmap-file-allocator-factory-dir") diff --git a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp index 5e0111cbe8c..b3d8e46cb3a 100644 --- a/searchlib/src/tests/diskindex/fusion/fusion_test.cpp +++ b/searchlib/src/tests/diskindex/fusion/fusion_test.cpp @@ -390,7 +390,7 @@ FusionTest::requireThatFusionIsWorking(const vespalib::string &prefix, bool dire ASSERT_TRUE(FileKit::hasStamp(tsName)); ASSERT_TRUE(FileKit::removeStamp(tsName)); ASSERT_FALSE(FileKit::hasStamp(tsName)); - vespalib::ThreadStackExecutor executor(4, 0x10000); + vespalib::ThreadStackExecutor executor(4); do { DiskIndex dw2(prefix + "dump2"); @@ -499,7 +499,7 @@ FusionTest::make_simple_index(const vespalib::string &dump_dir, const IFieldLeng bool FusionTest::try_merge_simple_indexes(const vespalib::string &dump_dir, const std::vector<vespalib::string> &sources, std::shared_ptr<IFlushToken> flush_token) { - vespalib::ThreadStackExecutor executor(4, 0x10000); + vespalib::ThreadStackExecutor executor(4); TuneFileIndexing tuneFileIndexing; DummyFileHeaderContext fileHeaderContext; SelectorArray selector(20, 0); diff --git a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp index 1df5f9a1409..efe858c016a 100644 --- a/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp +++ b/searchlib/src/tests/docstore/document_store_visitor/document_store_visitor_test.cpp @@ -240,7 +240,7 @@ Fixture::Fixture() _storeConfig(DocumentStore::Config(CompressionConfig::NONE, 0, 0), LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16_Ki))), - _executor(1, 128_Ki), + _executor(1), _fileHeaderContext(), _tlSyncer(), _store(), diff --git a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp index b295291d7c4..6e69e5092bc 100644 --- a/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp +++ b/searchlib/src/tests/docstore/file_chunk/file_chunk_test.cpp @@ -74,7 +74,7 @@ struct FixtureBase { explicit FixtureBase(const vespalib::string &baseName, bool dirCleanup = true) : dir(baseName), - executor(1, 0x10000), + executor(1), serialNum(1), tuneFile(), fileHeaderCtx(), diff --git a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp index 9538db391df..ad0bf60fbe3 100644 --- a/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp +++ b/searchlib/src/tests/docstore/logdatastore/logdatastore_test.cpp @@ -212,7 +212,7 @@ TEST("test that DirectIOPadding works accordng to spec") { void verifyGrowing(const LogDataStore::Config & config, uint32_t minFiles, uint32_t maxFiles) { DirectoryHandler tmpDir("growing"); - vespalib::ThreadStackExecutor executor(4, 128_Ki); + vespalib::ThreadStackExecutor executor(4); DummyFileHeaderContext fileHeaderContext; MyTlSyncer tlSyncer; { @@ -283,7 +283,7 @@ void fetchAndTest(IDataStore & datastore, uint32_t lid, const void *a, size_t sz TEST("testTruncatedIdxFile"){ LogDataStore::Config config; DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); MyTlSyncer tlSyncer; { // Files comes from the 'growing test'. @@ -311,7 +311,7 @@ TEST("testTruncatedIdxFile"){ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") { LogDataStore::Config config; DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "dangling-test", config, GrowStrategy(), TuneFileSummary(), @@ -324,7 +324,7 @@ TEST("testThatEmptyIdxFilesAndDanglingDatFilesAreRemoved") { TEST("testThatIncompleteCompactedFilesAreRemoved") { LogDataStore::Config config; DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "incompletecompact-test", config, GrowStrategy(), TuneFileSummary(), @@ -344,7 +344,7 @@ public: _myDir("visitcache"), _config(), _fileHeaderContext(), - _executor(1, 128_Ki), + _executor(1), _tlSyncer(), _datastore(_executor, _myDir.getDir(), _config, GrowStrategy(), TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr) @@ -533,7 +533,7 @@ VisitCacheStore::VisitCacheStore(UpdateStrategy strategy) : LogDataStore::Config().setMaxFileSize(50000).setMaxBucketSpread(3.0) .setFileConfig(WriteableFileChunk::Config(CompressionConfig(), 16_Ki))), _fileHeaderContext(), - _executor(1, 128_Ki), + _executor(1), _tlSyncer(), _datastore(std::make_unique<LogDocumentStore>(_executor, _myDir.getDir(), _config, GrowStrategy(), TuneFileSummary(), _fileHeaderContext, _tlSyncer, nullptr)), @@ -675,7 +675,7 @@ TEST("testWriteRead") { { std::filesystem::create_directory(std::filesystem::path("empty")); DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "empty", config, GrowStrategy(), TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); @@ -711,7 +711,7 @@ TEST("testWriteRead") { } { DummyFileHeaderContext fileHeaderContext; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); MyTlSyncer tlSyncer; LogDataStore datastore(executor, "empty", config, GrowStrategy(), TuneFileSummary(), @@ -762,7 +762,7 @@ TEST("requireThatFlushTimeIsAvailableAfterFlush") { vespalib::system_time before(vespalib::system_clock::now()); DummyFileHeaderContext fileHeaderContext; LogDataStore::Config config; - vespalib::ThreadStackExecutor executor(1, 128_Ki); + vespalib::ThreadStackExecutor executor(1); MyTlSyncer tlSyncer; LogDataStore store(executor, testDir.getDir(), config, GrowStrategy(), TuneFileSummary(), fileHeaderContext, tlSyncer, nullptr); @@ -849,7 +849,7 @@ struct Fixture { Fixture(const vespalib::string &dirName = "tmp", bool dirCleanup = true, size_t maxFileSize = 4_Ki * 2) - : executor(1, 0x20000), + : executor(1), dir(dirName), serialNum(0), fileHeaderCtx(), diff --git a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp index da900ddecee..b849143427c 100644 --- a/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp +++ b/searchlib/src/tests/docstore/store_by_bucket/store_by_bucket_test.cpp @@ -68,7 +68,7 @@ TEST("require that StoreByBucket gives bucket by bucket and ordered within") { std::mutex backing_lock; vespalib::MemoryDataStore backing(vespalib::alloc::Alloc::alloc(256), &backing_lock); - vespalib::ThreadStackExecutor executor(8, 128_Ki); + vespalib::ThreadStackExecutor executor(8); StoreByBucket sbb(backing, executor, CompressionConfig::LZ4); for (size_t i(1); i <=500; i++) { add(sbb, i); diff --git a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp index c1261a60278..69063f38aeb 100644 --- a/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp +++ b/searchlib/src/tests/memoryindex/memory_index/memory_index_test.cpp @@ -156,7 +156,7 @@ VESPA_THREAD_STACK_TAG(invert_executor) VESPA_THREAD_STACK_TAG(push_executor) Index::Index(const MySetup &setup) - : _executor(1, 128_Ki), + : _executor(1), _invertThreads(SequencedTaskExecutor::create(invert_executor, 2)), _pushThreads(SequencedTaskExecutor::create(push_executor, 2)), index(setup.make_all_index_schema(), setup, *_invertThreads, *_pushThreads), diff --git a/searchlib/src/tests/postinglistbm/stress_runner.cpp b/searchlib/src/tests/postinglistbm/stress_runner.cpp index 09272a60e75..179e4f49ef4 100644 --- a/searchlib/src/tests/postinglistbm/stress_runner.cpp +++ b/searchlib/src/tests/postinglistbm/stress_runner.cpp @@ -160,7 +160,7 @@ StressMaster::StressMaster(vespalib::Rand48 &rnd, { LOG(info, "StressMaster::StressMaster()"); - _threadPool = new FastOS_ThreadPool(128_Ki, 400); + _threadPool = new FastOS_ThreadPool(400); } StressMaster::~StressMaster() diff --git a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp index d8bfb1de9a9..ecf310798af 100644 --- a/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp +++ b/searchlib/src/tests/tensor/hnsw_index/stress_hnsw_mt.cpp @@ -18,7 +18,6 @@ #include <vespa/searchlib/tensor/random_level_generator.h> #include <vespa/searchlib/tensor/vector_bundle.h> #include <vespa/vespalib/data/input.h> -#include <vespa/vespalib/data/memory_input.h> #include <vespa/vespalib/data/slime/slime.h> #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vespalib/util/blockingthreadstackexecutor.h> @@ -254,8 +253,8 @@ public: vectors(), gen_handler(), index(), - multi_prepare_workers(10, 128_Ki, 50), - write_thread(1, 128_Ki, 500) + multi_prepare_workers(10, 50), + write_thread(1, 500) { loaded_vectors.load(); } diff --git a/searchlib/src/tests/transactionlog/translogclient_test.cpp b/searchlib/src/tests/transactionlog/translogclient_test.cpp index a922795d570..1eae8489c75 100644 --- a/searchlib/src/tests/transactionlog/translogclient_test.cpp +++ b/searchlib/src/tests/transactionlog/translogclient_test.cpp @@ -484,7 +484,7 @@ struct TLS { TransLogServer tls; TLS(const vespalib::string &name, int listenPort, const vespalib::string &baseDir, const common::FileHeaderContext &fileHeaderContext, const DomainConfig & cfg, size_t maxThreads = 4) - : threadPool(64_Ki), + : threadPool(), transport(), tls(transport, name, listenPort, baseDir, fileHeaderContext, cfg, maxThreads) { diff --git a/searchlib/src/tests/transactionlogstress/translogstress.cpp b/searchlib/src/tests/transactionlogstress/translogstress.cpp index 9988f3e171a..eb457f312e6 100644 --- a/searchlib/src/tests/transactionlogstress/translogstress.cpp +++ b/searchlib/src/tests/transactionlogstress/translogstress.cpp @@ -698,7 +698,7 @@ TransLogStress::main(int argc, char **argv) } // start transaction log server - FastOS_ThreadPool threadPool(256_Ki); + FastOS_ThreadPool threadPool; FNET_Transport transport; DummyFileHeaderContext fileHeaderContext; TransLogServer tls(transport, "server", 17897, ".", fileHeaderContext, DomainConfig().setPartSizeLimit(_cfg.domainPartSize)); diff --git a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp index a0ee40e6674..49fa7041533 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/domain.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/domain.cpp @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/destructor_callbacks.h> #include <vespa/vespalib/util/size_literals.h> #include <vespa/vespalib/util/retain_guard.h> +#include <vespa/vespalib/util/cpu_usage.h> #include <vespa/fastos/file.h> #include <algorithm> #include <thread> @@ -23,9 +24,12 @@ LOG_SETUP(".transactionlog.domain"); using vespalib::string; using vespalib::make_string_short::fmt; using vespalib::makeLambdaTask; +using vespalib::CpuUsage; using std::runtime_error; using std::make_shared; +using CpuCategory = vespalib::CpuUsage::Category; + namespace search::transactionlog { namespace { @@ -34,6 +38,7 @@ createCommitChunk(const DomainConfig &cfg) { return std::make_unique<CommitChunk>(cfg.getChunkSizeLimit(), cfg.getChunkSizeLimit()/256); } +VESPA_THREAD_STACK_TAG(tls_domain_commit); } Domain::Domain(const string &domainName, const string & baseDir, vespalib::Executor & executor, @@ -41,7 +46,7 @@ Domain::Domain(const string &domainName, const string & baseDir, vespalib::Execu : _config(cfg), _currentChunk(createCommitChunk(cfg)), _lastSerial(0), - _singleCommitter(std::make_unique<vespalib::ThreadStackExecutor>(1, 128_Ki)), + _singleCommitter(std::make_unique<vespalib::ThreadStackExecutor>(1, CpuUsage::wrap(tls_domain_commit, CpuCategory::WRITE))), _executor(executor), _sessionId(1), _name(domainName), diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp index b4534486e85..133fabd3e5f 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogclient.cpp @@ -47,7 +47,7 @@ struct RpcTask : public vespalib::Executor::Task { } TransLogClient::TransLogClient(FNET_Transport & transport, const vespalib::string & rpcTarget) : - _executor(std::make_unique<vespalib::ThreadStackExecutor>(1, 128_Ki, translogclient_rpc_callback)), + _executor(std::make_unique<vespalib::ThreadStackExecutor>(1, translogclient_rpc_callback)), _rpcTarget(rpcTarget), _sessions(), _supervisor(std::make_unique<FRT_Supervisor>(&transport)), diff --git a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp index a7aa2384bbc..c6ae9c00e49 100644 --- a/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp +++ b/searchlib/src/vespa/searchlib/transactionlog/translogserver.cpp @@ -98,8 +98,8 @@ TransLogServer::TransLogServer(FNET_Transport & transport, const vespalib::strin _name(name), _baseDir(baseDir), _domainConfig(cfg), - _executor(maxThreads, 128_Ki, CpuUsage::wrap(tls_executor, CpuUsage::Category::WRITE)), - _threadPool(std::make_unique<FastOS_ThreadPool>(120_Ki)), + _executor(maxThreads, CpuUsage::wrap(tls_executor, CpuUsage::Category::WRITE)), + _threadPool(std::make_unique<FastOS_ThreadPool>()), _supervisor(std::make_unique<FRT_Supervisor>(&transport)), _domains(), _reqQ(), |