From a45cc32c6f1b20b7307c42a779bbfae38414347d Mon Sep 17 00:00:00 2001 From: Henning Baldersheim Date: Mon, 22 Nov 2021 11:15:54 +0000 Subject: Get callback when setBucketStateCalculator has completed. --- .../document_subdbs/document_subdbs_test.cpp | 18 ++++++++++-------- .../searchcore/proton/attribute/attributemanager.cpp | 7 +++++-- .../searchcore/proton/attribute/attributemanager.h | 2 +- .../proton/attribute/filter_attribute_manager.cpp | 7 +++++-- .../proton/attribute/filter_attribute_manager.h | 2 +- .../searchcore/proton/attribute/i_attribute_manager.h | 4 ++-- .../src/vespa/searchcore/proton/server/documentdb.cpp | 2 +- .../proton/server/documentsubdbcollection.cpp | 4 ++-- .../searchcore/proton/server/documentsubdbcollection.h | 4 +++- .../searchcore/proton/server/fast_access_doc_subdb.cpp | 5 ++++- .../vespa/searchcore/proton/server/idocumentsubdb.h | 4 +++- .../searchcore/proton/server/searchabledocsubdb.cpp | 4 ++-- .../searchcore/proton/server/searchabledocsubdb.h | 2 +- .../searchcore/proton/server/storeonlydocsubdb.cpp | 8 ++++---- .../vespa/searchcore/proton/server/storeonlydocsubdb.h | 4 ++-- .../searchcore/proton/test/dummy_document_sub_db.h | 2 +- .../searchcore/proton/test/mock_attribute_manager.h | 2 +- 17 files changed, 48 insertions(+), 33 deletions(-) diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 144e37d9461..a9faeceab48 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -316,6 +316,11 @@ struct FixtureBase _writeService.master().execute(makeLambdaTask([this]() { _subDb.close(); })); _writeService.shutdown(); } + void setBucketStateCalculator(const std::shared_ptr & calc) { + vespalib::Gate gate; + _subDb.setBucketStateCalculator(calc, std::make_shared(gate)); + gate.await(); + } template void runInMasterAndSync(FunctionType func) { proton::test::runInMasterAndSync(_writeService, func); @@ -336,7 +341,7 @@ struct FixtureBase runInMasterAndSync([&]() { performReconfig(serialNum, reconfigSchema, reconfigConfigDir); }); } void performReconfig(SerialNum serialNum, const Schema &reconfigSchema, const vespalib::string &reconfigConfigDir) { - MyConfigSnapshot::UP newCfg(new MyConfigSnapshot(reconfigSchema, reconfigConfigDir)); + auto newCfg = std::make_unique(reconfigSchema, reconfigConfigDir); DocumentDBConfig::ComparisonResult cmpResult; cmpResult.attributesChanged = true; cmpResult.documenttypesChanged = true; @@ -564,14 +569,14 @@ TEST_F("require that subdb reflect retirement", FastAccessFixture) auto calc = std::make_shared(); calc->setNodeRetired(true); - f._subDb.setBucketStateCalculator(calc); + f.setBucketStateCalculator(calc); EXPECT_TRUE(f._subDb.isNodeRetired()); auto retired_cfg = f._subDb.computeCompactionStrategy(cfg); EXPECT_TRUE(cfg != retired_cfg); EXPECT_TRUE(search::CompactionStrategy(0.5, 0.5) == retired_cfg); calc->setNodeRetired(false); - f._subDb.setBucketStateCalculator(calc); + f.setBucketStateCalculator(calc); EXPECT_FALSE(f._subDb.isNodeRetired()); unretired_cfg = f._subDb.computeCompactionStrategy(cfg); EXPECT_TRUE(cfg == unretired_cfg); @@ -587,21 +592,18 @@ TEST_F("require that attribute compaction config reflect retirement", FastAccess auto calc = std::make_shared(); calc->setNodeRetired(true); - f._subDb.setBucketStateCalculator(calc); - f._writeService.sync_all_executors(); + f.setBucketStateCalculator(calc); guard = f._subDb.getAttributeManager()->getAttribute("attr1"); EXPECT_EQUAL(retired_cfg, (*guard)->getConfig().getCompactionStrategy()); EXPECT_EQUAL(retired_cfg, dynamic_cast(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy()); f.basicReconfig(10); - f._writeService.sync_all_executors(); guard = f._subDb.getAttributeManager()->getAttribute("attr1"); EXPECT_EQUAL(retired_cfg, (*guard)->getConfig().getCompactionStrategy()); EXPECT_EQUAL(retired_cfg, dynamic_cast(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy()); calc->setNodeRetired(false); - f._subDb.setBucketStateCalculator(calc); - f._writeService.sync_all_executors(); + f.setBucketStateCalculator(calc); guard = f._subDb.getAttributeManager()->getAttribute("attr1"); EXPECT_EQUAL(default_cfg, (*guard)->getConfig().getCompactionStrategy()); EXPECT_EQUAL(default_cfg, dynamic_cast(f._subDb.getDocumentMetaStoreContext().get()).getConfig().getCompactionStrategy()); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index 5ee162e9264..eee6264b9f4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -595,7 +595,7 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr } void -AttributeManager::asyncForEachAttribute(std::shared_ptr func) const +AttributeManager::asyncForEachAttribute(std::shared_ptr func, OnDone onDone) const { for (const auto &attr : _attributes) { if (attr.second.isExtra()) { @@ -604,7 +604,10 @@ AttributeManager::asyncForEachAttribute(std::shared_ptr func) } AttributeVector::SP attrsp = attr.second.getAttribute(); _attributeFieldWriter.execute(_attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()), - [attrsp, func]() { (*func)(*attrsp); }); + [attrsp, func, onDone]() { + (void) onDone; + (*func)(*attrsp); + }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index e2b9550435d..08e2d511d70 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -178,7 +178,7 @@ public: const std::vector &getWritableAttributes() const override; void asyncForEachAttribute(std::shared_ptr func) const override; - void asyncForEachAttribute(std::shared_ptr func) const override; + void asyncForEachAttribute(std::shared_ptr func, OnDone onDone) const override; void asyncForAttribute(const vespalib::string &name, std::unique_ptr func) const override; ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp index c7ab83ae590..5f162281d96 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.cpp @@ -206,7 +206,7 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr func) const +FilterAttributeManager::asyncForEachAttribute(std::shared_ptr func, OnDone onDone) const { // Run by document db master thread std::vector completeList; @@ -217,7 +217,10 @@ FilterAttributeManager::asyncForEachAttribute(std::shared_ptr // Name must be extracted in document db master thread or attribute // writer thread attributeFieldWriter.execute(attributeFieldWriter.getExecutorIdFromName(attrsp->getNamePrefix()), - [attrsp, func]() { (*func)(*attrsp); }); + [attrsp, func, onDone]() { + (void) onDone; + (*func)(*attrsp); + }); } } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h index 1ae5f452218..1512ab32d62 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/filter_attribute_manager.h @@ -52,7 +52,7 @@ public: search::AttributeVector * getWritableAttribute(const vespalib::string &name) const override; const std::vector & getWritableAttributes() const override; void asyncForEachAttribute(std::shared_ptr func) const override; - void asyncForEachAttribute(std::shared_ptr func) const override; + void asyncForEachAttribute(std::shared_ptr func, OnDone onDone) const override; ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const override; void setImportedAttributes(std::unique_ptr attributes) override; const ImportedAttributesRepo *getImportedAttributes() const override; diff --git a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h index d55cd45d014..b8968ba9d2e 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/i_attribute_manager.h @@ -31,7 +31,7 @@ class ImportedAttributesRepo; struct IAttributeManager : public search::IAttributeManager { using SP = std::shared_ptr; - using OnWriteDoneType = const std::shared_ptr &; + using OnDone = std::shared_ptr; using IAttributeFunctor = search::attribute::IAttributeFunctor; using IConstAttributeFunctor = search::attribute::IConstAttributeFunctor; @@ -98,7 +98,7 @@ struct IAttributeManager : public search::IAttributeManager virtual const std::vector &getWritableAttributes() const = 0; virtual void asyncForEachAttribute(std::shared_ptr func) const = 0; - virtual void asyncForEachAttribute(std::shared_ptr func) const = 0; + virtual void asyncForEachAttribute(std::shared_ptr func, OnDone onDone) const = 0; virtual ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &name) const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 142a2f748a2..f8410322fed 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -997,7 +997,7 @@ DocumentDB::notifyClusterStateChanged(const std::shared_ptrsetCalculator(newCalc); } - _subDBs.setBucketStateCalculator(newCalc); + _subDBs.setBucketStateCalculator(newCalc, std::shared_ptr()); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 1ec639a2e21..6576e4ead97 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -303,11 +303,11 @@ DocumentSubDBCollection::close() } void -DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc) +DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc, OnDone onDone) { _calc = calc; for (auto subDb : _subDBs) { - subDb->setBucketStateCalculator(calc); + subDb->setBucketStateCalculator(calc, onDone); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index d309593681c..515a886969c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -7,6 +7,7 @@ #include #include #include +#include #include namespace vespalib { @@ -58,6 +59,7 @@ public: using SubDBVector = std::vector; using const_iterator = SubDBVector::const_iterator; using SerialNum = search::SerialNum; + using OnDone = std::shared_ptr; private: using IFeedViewSP = std::shared_ptr; @@ -95,7 +97,7 @@ public: const HwInfo &hwInfo); ~DocumentSubDBCollection(); - void setBucketStateCalculator(const IBucketStateCalculatorSP &calc); + void setBucketStateCalculator(const IBucketStateCalculatorSP &calc, OnDone onDone); void createRetrievers(); void maintenanceSync(MaintenanceController &mc); diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 40a1a1a45f3..f8a6253ead1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -276,7 +276,10 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const } _iFeedView.set(_fastAccessFeedView.get()); if (isNodeRetired()) { - reconfigureAttributesConsideringNodeState(); + // TODO Should probably ahve a similar OnDone callback to applyConfig too. + vespalib::Gate gate; + reconfigureAttributesConsideringNodeState(std::make_shared(gate)); + gate.await(); } } return tasks; diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 00849d6ad31..f84352a4558 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace search::index { class Schema; } @@ -60,6 +61,7 @@ public: using SchemaSP = std::shared_ptr; using IFlushTargetList = std::vector>; using IndexConfig = index::IndexConfig; + using OnDone = std::shared_ptr; public: IDocumentSubDB() { } virtual ~IDocumentSubDB() { } @@ -77,7 +79,7 @@ public: virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) = 0; - virtual void setBucketStateCalculator(const std::shared_ptr &calc) = 0; + virtual void setBucketStateCalculator(const std::shared_ptr &calc, OnDone) = 0; virtual std::shared_ptr getSearchView() const = 0; virtual std::shared_ptr getFeedView() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 0aaf0bfb06f..6e87f33e1c6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -180,9 +180,9 @@ SearchableDocSubDB::propagateFlushConfig() } void -SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr &calc) +SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr &calc, OnDone onDone) { - FastAccessDocSubDB::setBucketStateCalculator(calc); + FastAccessDocSubDB::setBucketStateCalculator(calc, std::move(onDone)); propagateFlushConfig(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 376a319c7e7..ea758443cd4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -105,7 +105,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; - void setBucketStateCalculator(const std::shared_ptr &calc) override; + void setBucketStateCalculator(const std::shared_ptr &calc, OnDone onDone) override; void clearViews() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 97e55c37aff..06d174497b3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -460,7 +460,7 @@ StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, } void -StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr & calc) { +StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr & calc, OnDone onDone) { bool wasNodeRetired = isNodeRetired(); _nodeRetired = calc->nodeRetired(); if (wasNodeRetired != isNodeRetired()) { @@ -468,16 +468,16 @@ StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptrgetConfig(); cfg.setCompactionStrategy(compactionStrategy); _dms->update_config(cfg); - reconfigureAttributesConsideringNodeState(); + reconfigureAttributesConsideringNodeState(std::move(onDone)); } } void -StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState() { +StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState(OnDone onDone) { search::CompactionStrategy compactionStrategy = computeCompactionStrategy(_lastConfiguredCompactionStrategy); auto attrMan = getAttributeManager(); if (attrMan) { - attrMan->asyncForEachAttribute(std::make_shared(compactionStrategy)); + attrMan->asyncForEachAttribute(std::make_shared(compactionStrategy), std::move(onDone)); } } diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index c7eaa0ad0b1..b53dfe89f59 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -183,7 +183,7 @@ protected: StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams(); vespalib::string getSubDbName() const; void reconfigure(const search::LogDocumentStore::Config & protonConfig, const AllocStrategy& alloc_strategy); - void reconfigureAttributesConsideringNodeState(); + void reconfigureAttributesConsideringNodeState(OnDone onDone); public: StoreOnlyDocSubDB(const Config &cfg, const Context &ctx); ~StoreOnlyDocSubDB() override; @@ -203,7 +203,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; - void setBucketStateCalculator(const std::shared_ptr &calc) override; + void setBucketStateCalculator(const std::shared_ptr &calc, OnDone onDone) override; ISearchHandler::SP getSearchView() const override { return _iSearchView.get(); } IFeedView::SP getFeedView() const override { return _iFeedView.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 5cc0ac5a186..03ddcf3605b 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -58,7 +58,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB { return IReprocessingTask::List(); } - void setBucketStateCalculator(const std::shared_ptr &) override { } + void setBucketStateCalculator(const std::shared_ptr &, OnDone) override { } ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); } IFeedView::SP getFeedView() const override { return IFeedView::SP(); } void clearViews() override {} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h index abc8eb679dd..743cd9af8fc 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_attribute_manager.h @@ -87,7 +87,7 @@ public: return _writables; } void asyncForEachAttribute(std::shared_ptr) const override { } - void asyncForEachAttribute(std::shared_ptr) const override { } + void asyncForEachAttribute(std::shared_ptr, OnDone) const override { } ExclusiveAttributeReadAccessor::UP getExclusiveReadAccessor(const vespalib::string &) const override { return ExclusiveAttributeReadAccessor::UP(); -- cgit v1.2.3