diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2019-05-14 14:02:19 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2019-05-14 18:29:04 +0000 |
commit | d2dec4cfb9cff2c746ba20e727c6f8c846c7985d (patch) | |
tree | e624f734c4d59313f037ffaf0ba178d942b90747 /searchcore | |
parent | 67f55d7c0c14fee4830a010281fc0c2c752c1d8a (diff) |
Wire in the shared executor in the threading service.
Diffstat (limited to 'searchcore')
16 files changed, 72 insertions, 33 deletions
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 87853bb414d..5172489ced1 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -88,6 +88,7 @@ struct ViewSet { IndexManagerDummyReconfigurer _reconfigurer; DummyFileHeaderContext _fileHeaderContext; + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _writeService; SearchableFeedView::SerialNum serialNum; std::shared_ptr<const DocumentTypeRepo> repo; @@ -117,7 +118,8 @@ struct ViewSet ViewSet::ViewSet() : _reconfigurer(), _fileHeaderContext(), - _writeService(), + _sharedExecutor(1, 0x10000), + _writeService(_sharedExecutor), serialNum(1), repo(createRepo()), _docTypeName(DOC_TYPE), @@ -285,11 +287,13 @@ MyFastAccessFeedView::~MyFastAccessFeedView() = default; struct FastAccessFixture { + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _writeService; MyFastAccessFeedView _view; FastAccessDocSubDBConfigurer _configurer; FastAccessFixture() - : _writeService(), + : _sharedExecutor(1, 0x10000), + _writeService(_sharedExecutor), _view(_writeService), _configurer(_view._feedView, IAttributeWriterFactory::UP(new AttributeWriterFactory), "test") { diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 32cf0584768..cbf82fd7e11 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -292,8 +292,8 @@ struct MyConfigSnapshot template <typename Traits> struct FixtureBase { - ExecutorThreadingService _writeService; ThreadStackExecutor _summaryExecutor; + ExecutorThreadingService _writeService; typename Traits::Config _cfg; std::shared_ptr<BucketDBOwner> _bucketDB; BucketDBHandler _bucketDBHandler; @@ -304,8 +304,8 @@ struct FixtureBase typename Traits::SubDB _subDb; IFeedView::SP _tmpFeedView; FixtureBase() - : _writeService(), - _summaryExecutor(1, 64 * 1024), + : _summaryExecutor(1, 64 * 1024), + _writeService(_summaryExecutor), _cfg(), _bucketDB(std::make_shared<BucketDBOwner>()), _bucketDBHandler(*_bucketDB), diff --git a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp index caee69b311e..39fc93d5725 100644 --- a/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedhandler/feedhandler_test.cpp @@ -456,6 +456,7 @@ struct FeedHandlerFixture DummyFileHeaderContext _fileHeaderContext; TransLogServer tls; vespalib::string tlsSpec; + vespalib::ThreadStackExecutor sharedExecutor; ExecutorThreadingService writeService; SchemaContext schema; MyOwner owner; @@ -471,7 +472,8 @@ struct FeedHandlerFixture : _fileHeaderContext(), tls("mytls", 9016, "mytlsdir", _fileHeaderContext, 0x10000), tlsSpec("tcp/localhost:9016"), - writeService(), + sharedExecutor(1, 0x10000), + writeService(sharedExecutor), schema(), owner(), _state(), diff --git a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp index fd1984a79fe..cfc3e4373a1 100644 --- a/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/feedview/feedview_test.cpp @@ -519,6 +519,7 @@ struct FixtureBase DocumentMetaStoreContext::SP _dmscReal; test::DocumentMetaStoreContextObserver::SP _dmsc; ParamsContext pc; + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _writeServiceReal; test::ThreadingServiceObserver _writeService; documentmetastore::LidReuseDelayer _lidReuseDelayer; @@ -703,7 +704,8 @@ FixtureBase::FixtureBase(TimeStamp visibilityDelay) _dmscReal(new DocumentMetaStoreContext(std::make_shared<BucketDBOwner>())), _dmsc(new test::DocumentMetaStoreContextObserver(*_dmscReal)), pc(sc._builder->getDocumentType().getName(), "fileconfig_test"), - _writeServiceReal(), + _sharedExecutor(1, 0x10000), + _writeServiceReal(_sharedExecutor), _writeService(_writeServiceReal), _lidReuseDelayer(_writeService, _dmsc->get()), _commitTimeTracker(visibilityDelay), diff --git a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp index d07c29ead69..b11525bae3d 100644 --- a/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp +++ b/searchcore/src/tests/proton/documentdb/storeonlyfeedview/storeonlyfeedview_test.cpp @@ -201,6 +201,7 @@ struct FixtureBase { int heartbeatCount; int outstandingMoveOps; DocumentMetaStore::SP metaStore; + vespalib::ThreadStackExecutor sharedExecutor; ExecutorThreadingService writeService; documentmetastore::LidReuseDelayer lidReuseDelayer; CommitTimeTracker commitTimeTracker; @@ -214,10 +215,10 @@ struct FixtureBase { metaStore(new DocumentMetaStore(std::make_shared<BucketDBOwner>(), DocumentMetaStore::getFixedName(), search::GrowStrategy(), - DocumentMetaStore::IGidCompare::SP( - new DocumentMetaStore::DefaultGidCompare), + std::make_shared<DocumentMetaStore::DefaultGidCompare>(), subDbType)), - writeService(), + sharedExecutor(1, 0x10000), + writeService(sharedExecutor), lidReuseDelayer(writeService, *metaStore), commitTimeTracker(fastos::TimeStamp()), feedview() diff --git a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp index 89764a562e6..d305751f7c2 100644 --- a/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp +++ b/searchcore/src/tests/proton/documentmetastore/lidreusedelayer/lidreusedelayer_test.cpp @@ -143,13 +143,15 @@ public: class Fixture { public: + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _writeServiceReal; test::ThreadingServiceObserver _writeService; MyMetaStore _store; documentmetastore::LidReuseDelayer _lidReuseDelayer; Fixture() - : _writeServiceReal(), + : _sharedExecutor(1, 0x10000), + _writeServiceReal(_sharedExecutor), _writeService(_writeServiceReal), _store(), _lidReuseDelayer(_writeService, _store) diff --git a/searchcore/src/tests/proton/index/fusionrunner_test.cpp b/searchcore/src/tests/proton/index/fusionrunner_test.cpp index 9a1ddee4278..8f9944e178e 100644 --- a/searchcore/src/tests/proton/index/fusionrunner_test.cpp +++ b/searchcore/src/tests/proton/index/fusionrunner_test.cpp @@ -62,6 +62,7 @@ class Test : public vespalib::TestApp { FixedSourceSelector::UP _selector; FusionSpec _fusion_spec; DummyFileHeaderContext _fileHeaderContext; + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _threadingService; IndexManager::MaintainerOperations _ops; @@ -84,7 +85,8 @@ public: _selector(), _fusion_spec(), _fileHeaderContext(), - _threadingService(), + _sharedExecutor(1, 0x10000), + _threadingService(_sharedExecutor), _ops(_fileHeaderContext, TuneFileIndexManager(), 0, _threadingService) diff --git a/searchcore/src/tests/proton/index/indexmanager_test.cpp b/searchcore/src/tests/proton/index/indexmanager_test.cpp index 841c69289bf..9abd3e6cb39 100644 --- a/searchcore/src/tests/proton/index/indexmanager_test.cpp +++ b/searchcore/src/tests/proton/index/indexmanager_test.cpp @@ -100,6 +100,7 @@ struct Fixture { SerialNum _serial_num; IndexManagerDummyReconfigurer _reconfigurer; DummyFileHeaderContext _fileHeaderContext; + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _writeService; std::unique_ptr<IndexManager> _index_manager; Schema _schema; @@ -109,7 +110,8 @@ struct Fixture { : _serial_num(0), _reconfigurer(), _fileHeaderContext(), - _writeService(), + _sharedExecutor(1, 0x10000), + _writeService(_sharedExecutor), _index_manager(), _schema(getSchema()), _builder(_schema) diff --git a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp index 5b5bea412f7..33ee3eb6105 100644 --- a/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp +++ b/searchcore/src/tests/proton/server/visibility_handler/visibility_handler_test.cpp @@ -63,6 +63,7 @@ class Fixture { public: MyGetSerialNum _getSerialNum; + vespalib::ThreadStackExecutor _sharedExecutor; ExecutorThreadingService _writeServiceReal; ThreadingServiceObserver _writeService; std::shared_ptr<MyFeedView> _feedViewReal; @@ -72,7 +73,8 @@ public: Fixture() : _getSerialNum(), - _writeServiceReal(), + _sharedExecutor(1, 0x10000), + _writeServiceReal(_sharedExecutor), _writeService(_writeServiceReal), _feedViewReal(std::make_shared<MyFeedView>()), _feedView(_feedViewReal), diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp index c17731efbc5..7a1e2213a52 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.cpp @@ -7,12 +7,14 @@ namespace proton { ExecutorThreadingServiceStats::ExecutorThreadingServiceStats(Stats masterExecutorStats, Stats indexExecutorStats, Stats summaryExecutorStats, + Stats sharedExecutorStats, Stats indexFieldInverterExecutorStats, Stats indexFieldWriterExecutorStats, Stats attributeFieldWriterExecutorStats) : _masterExecutorStats(masterExecutorStats), _indexExecutorStats(indexExecutorStats), _summaryExecutorStats(summaryExecutorStats), + _sharedExecutorStats(sharedExecutorStats), _indexFieldInverterExecutorStats(indexFieldInverterExecutorStats), _indexFieldWriterExecutorStats(indexFieldWriterExecutorStats), _attributeFieldWriterExecutorStats(attributeFieldWriterExecutorStats) diff --git a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h index 7fbff6e5eb0..ea0e989374d 100644 --- a/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h +++ b/searchcore/src/vespa/searchcore/proton/metrics/executor_threading_service_stats.h @@ -18,6 +18,7 @@ private: Stats _masterExecutorStats; Stats _indexExecutorStats; Stats _summaryExecutorStats; + Stats _sharedExecutorStats; Stats _indexFieldInverterExecutorStats; Stats _indexFieldWriterExecutorStats; Stats _attributeFieldWriterExecutorStats; @@ -25,6 +26,7 @@ public: ExecutorThreadingServiceStats(Stats masterExecutorStats, Stats indexExecutorStats, Stats summaryExecutorStats, + Stats sharedExecutorStats, Stats indexFieldInverterExecutorStats, Stats indexFieldWriterExecutorStats, Stats attributeFieldWriterExecutorStats); @@ -33,6 +35,7 @@ public: const Stats &getMasterExecutorStats() const { return _masterExecutorStats; } const Stats &getIndexExecutorStats() const { return _indexExecutorStats; } const Stats &getSummaryExecutorStats() const { return _summaryExecutorStats; } + const Stats &getSharedExecutorStats() const { return _sharedExecutorStats; } const Stats &getIndexFieldInverterExecutorStats() const { return _indexFieldInverterExecutorStats; } const Stats &getIndexFieldWriterExecutorStats() const { return _indexFieldWriterExecutorStats; } const Stats &getAttributeFieldWriterExecutorStats() const { return _attributeFieldWriterExecutorStats; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 6158dde71de..1d0429ee81a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -140,7 +140,8 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, ThreadingServiceConfig::make(protonCfg, findDocumentDB(protonCfg.documentdb, docTypeName.getName())->feeding.concurrency, hwInfo.cpu())), - _writeService(_writeServiceConfig.indexingThreads(), + _writeService(sharedExecutor, + _writeServiceConfig.indexingThreads(), indexing_thread_stack_size, _writeServiceConfig.defaultTaskLimit()), _initializeThreads(std::move(initializeThreads)), diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp index 3d9525f0471..f971b0dcce7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.cpp @@ -9,14 +9,17 @@ using search::SequencedTaskExecutor; namespace proton { -ExecutorThreadingService::ExecutorThreadingService(uint32_t threads, uint32_t stackSize, uint32_t taskLimit) +ExecutorThreadingService::ExecutorThreadingService(vespalib::ThreadStackExecutorBase & sharedExecutor, + uint32_t threads, uint32_t stackSize, uint32_t taskLimit) - : _masterExecutor(1, stackSize), + : _sharedExecutor(sharedExecutor), + _masterExecutor(1, stackSize), _indexExecutor(1, stackSize, taskLimit), _summaryExecutor(1, stackSize, taskLimit), _masterService(_masterExecutor), _indexService(_indexExecutor), _summaryService(_summaryExecutor), + _sharedService(_sharedExecutor), _indexFieldInverter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)), _indexFieldWriter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)), _attributeFieldWriter(std::make_unique<SequencedTaskExecutor>(threads, taskLimit)) @@ -73,6 +76,7 @@ ExecutorThreadingService::getStats() return ExecutorThreadingServiceStats(_masterExecutor.getStats(), _indexExecutor.getStats(), _summaryExecutor.getStats(), + _sharedExecutor.getStats(), _indexFieldInverter->getStats(), _indexFieldWriter->getStats(), _attributeFieldWriter->getStats()); diff --git a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h index 240eefab2a7..1e63aa11df5 100644 --- a/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h +++ b/searchcore/src/vespa/searchcore/proton/server/executorthreadingservice.h @@ -18,12 +18,14 @@ class ExecutorThreadingServiceStats; class ExecutorThreadingService : public searchcorespi::index::IThreadingService { private: + vespalib::ThreadStackExecutorBase & _sharedExecutor; vespalib::ThreadStackExecutor _masterExecutor; vespalib::BlockingThreadStackExecutor _indexExecutor; vespalib::BlockingThreadStackExecutor _summaryExecutor; ExecutorThreadService _masterService; ExecutorThreadService _indexService; ExecutorThreadService _summaryService; + ExecutorThreadService _sharedService; std::unique_ptr<search::SequencedTaskExecutor> _indexFieldInverter; std::unique_ptr<search::SequencedTaskExecutor> _indexFieldWriter; std::unique_ptr<search::SequencedTaskExecutor> _attributeFieldWriter; @@ -35,7 +37,8 @@ public: * @stackSize The size of the stack of the underlying executors. * @taskLimit The task limit for the index executor. */ - ExecutorThreadingService(uint32_t threads = 1, + ExecutorThreadingService(vespalib::ThreadStackExecutorBase &sharedExecutor, + uint32_t threads = 1, uint32_t stackSize = 128 * 1024, uint32_t taskLimit = 1000); ~ExecutorThreadingService() override; @@ -59,6 +62,9 @@ public: vespalib::ThreadStackExecutorBase &getSummaryExecutor() { return _summaryExecutor; } + vespalib::ThreadStackExecutorBase &getSharedExecutor() { + return _sharedExecutor; + } /** * Implements IThreadingService @@ -73,6 +79,9 @@ public: searchcorespi::index::IThreadService &summary() override { return _summaryService; } + searchcorespi::index::IThreadService &shared() override { + return _sharedService; + } search::ISequencedTaskExecutor &indexFieldInverter() override; search::ISequencedTaskExecutor &indexFieldWriter() override; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 69e784a8215..6c11ebbbf6b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -26,6 +26,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB IIndexManager::SP _indexManager; ISummaryAdapter::SP _summaryAdapter; IIndexWriter::SP _indexWriter; + vespalib::ThreadStackExecutor _sharedExecutor; std::unique_ptr<ExecutorThreadingService> _writeService; DummyDocumentSubDb(std::shared_ptr<BucketDBOwner> bucketDB, uint32_t subDbId) @@ -35,7 +36,8 @@ struct DummyDocumentSubDb : public IDocumentSubDB _indexManager(), _summaryAdapter(), _indexWriter(), - _writeService(std::make_unique<ExecutorThreadingService>(1)) + _sharedExecutor(1, 0x10000), + _writeService(std::make_unique<ExecutorThreadingService>(_sharedExecutor, 1)) { } ~DummyDocumentSubDb() {} diff --git a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h index ac6b44dea5f..750d6160a84 100644 --- a/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h +++ b/searchcore/src/vespa/searchcore/proton/test/threading_service_observer.h @@ -6,8 +6,7 @@ #include <vespa/searchcorespi/index/ithreadingservice.h> #include <vespa/searchlib/common/sequencedtaskexecutorobserver.h> -namespace proton { -namespace test { +namespace proton:: test { class ThreadingServiceObserver : public searchcorespi::index::IThreadingService { @@ -16,6 +15,7 @@ private: ThreadServiceObserver _master; ThreadServiceObserver _index; ThreadServiceObserver _summary; + ThreadServiceObserver _shared; search::SequencedTaskExecutorObserver _indexFieldInverter; search::SequencedTaskExecutorObserver _indexFieldWriter; search::SequencedTaskExecutorObserver _attributeFieldWriter; @@ -26,12 +26,13 @@ public: _master(_service.master()), _index(service.index()), _summary(service.summary()), + _shared(service.shared()), _indexFieldInverter(_service.indexFieldInverter()), _indexFieldWriter(_service.indexFieldWriter()), _attributeFieldWriter(_service.attributeFieldWriter()) { } - virtual ~ThreadingServiceObserver() override { } + ~ThreadingServiceObserver() override { } const ThreadServiceObserver &masterObserver() const { return _master; } @@ -55,35 +56,35 @@ public: /** * Implements vespalib::Syncable */ - virtual vespalib::Syncable &sync() override { + vespalib::Syncable &sync() override { return _service.sync(); } /** * Implements IThreadingService */ - virtual searchcorespi::index::IThreadService &master() override { + searchcorespi::index::IThreadService &master() override { return _master; } - virtual searchcorespi::index::IThreadService &index() override { + searchcorespi::index::IThreadService &index() override { return _index; } - virtual searchcorespi::index::IThreadService &summary() override { + searchcorespi::index::IThreadService &summary() override { return _summary; } - virtual search::ISequencedTaskExecutor &indexFieldInverter() override { + searchcorespi::index::IThreadService &shared() override { + return _shared; + } + search::ISequencedTaskExecutor &indexFieldInverter() override { return _indexFieldInverter; } - virtual search::ISequencedTaskExecutor &indexFieldWriter() override { + search::ISequencedTaskExecutor &indexFieldWriter() override { return _indexFieldWriter; } - virtual search::ISequencedTaskExecutor &attributeFieldWriter() override { + search::ISequencedTaskExecutor &attributeFieldWriter() override { return _attributeFieldWriter; } }; -} // namespace test -} // namespace proton - - +} |