diff options
Diffstat (limited to 'searchcore/src/vespa/searchcore/proton/server')
5 files changed, 67 insertions, 15 deletions
diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 5bd9ba64bae..8451f3268b8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -10,13 +10,11 @@ #include <vespa/searchcore/proton/attribute/attribute_collection_spec_factory.h> #include <vespa/searchcore/proton/attribute/attribute_factory.h> #include <vespa/searchcore/proton/attribute/attribute_manager_initializer.h> -#include <vespa/searchcore/proton/attribute/attribute_populator.h> #include <vespa/searchcore/proton/attribute/filter_attribute_manager.h> #include <vespa/searchcore/proton/attribute/sequential_attributes_initializer.h> #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/matching/sessionmanager.h> #include <vespa/searchcore/proton/reprocessing/attribute_reprocessing_initializer.h> -#include <vespa/searchcore/proton/reprocessing/document_reprocessing_handler.h> #include <vespa/searchcore/proton/reprocessing/reprocess_documents_task.h> #include <vespa/searchlib/docstore/document_store_visitor_progress.h> #include <vespa/log/log.h> @@ -275,6 +273,9 @@ FastAccessDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const reconfigureAttributeMetrics(*newMgr, *oldMgr); } _iFeedView.set(_fastAccessFeedView.get()); + if (isNodeRetired()) { + reconfigureAttributesConsideringNodeState(); + } } return tasks; } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 3322722a642..b512143c5fd 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -5,7 +5,6 @@ #include "document_subdb_initializer.h" #include "reconfig_params.h" #include "i_document_subdb_owner.h" -#include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> @@ -43,8 +42,7 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()), _warmupExecutor(ctx._warmupExecutor), _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()), - _flushConfig(), - _nodeRetired(false) + _flushConfig() { _gidToLidChangeHandler = _realGidToLidChangeHandler; } @@ -177,14 +175,14 @@ SearchableDocSubDB::applyFlushConfig(const DocumentDBFlushConfig &flushConfig) void SearchableDocSubDB::propagateFlushConfig() { - uint32_t maxFlushed = _nodeRetired ? _flushConfig.getMaxFlushedRetired() : _flushConfig.getMaxFlushed(); + uint32_t maxFlushed = isNodeRetired() ? _flushConfig.getMaxFlushedRetired() : _flushConfig.getMaxFlushed(); _indexMgr->setMaxFlushed(maxFlushed); } void SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) { - _nodeRetired = calc->nodeRetired(); + FastAccessDocSubDB::setBucketStateCalculator(calc); propagateFlushConfig(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index c310aeb2a2b..2e7aac0a8d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -81,7 +81,6 @@ private: vespalib::SyncableThreadExecutor &_warmupExecutor; std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; DocumentDBFlushConfig _flushConfig; - bool _nodeRetired; // Note: lifetime of indexManager must be handled by caller. std::shared_ptr<initializer::InitializerTask> diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 7ab60270411..97e55c37aff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "storeonlydocsubdb.h" #include "docstorevalidator.h" #include "document_subdb_initializer.h" #include "document_subdb_initializer_result.h" @@ -7,7 +8,7 @@ #include "i_document_subdb_owner.h" #include "minimal_document_retriever.h" #include "reconfig_params.h" -#include "storeonlydocsubdb.h" +#include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/bucketdb/ibucketdbhandlerinitializer.h> #include <vespa/searchcore/proton/common/alloc_config.h> @@ -111,6 +112,8 @@ StoreOnlyDocSubDB::StoreOnlyDocSubDB(const Config &cfg, const Context &ctx) _dmsFlushTarget(), _dmsShrinkTarget(), _pendingLidsForCommit(std::make_shared<PendingLidTracker>()), + _nodeRetired(false), + _lastConfiguredCompactionStrategy(), _subDbId(cfg._subDbId), _subDbType(cfg._subDbType), _fileHeaderContext(ctx._fileHeaderContext, _docTypeName, _baseDir), @@ -280,6 +283,7 @@ StoreOnlyDocSubDB::setupDocumentMetaStore(DocumentMetaStoreInitializerResult::SP _dmsShrinkTarget = std::make_shared<ShrinkLidSpaceFlushTarget>("documentmetastore.shrink", Type::GC, Component::ATTRIBUTE, _flushedDocumentMetaStoreSerialNum, _dmsFlushTarget->getLastFlushTime(), dms); + _lastConfiguredCompactionStrategy = dms->getConfig().getCompactionStrategy(); } DocumentSubDbInitializer::UP @@ -413,22 +417,68 @@ StoreOnlyDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const return IReprocessingTask::List(); } +namespace { + +constexpr double RETIRED_DEAD_RATIO = 0.5; + +struct UpdateConfig : public search::attribute::IAttributeFunctor { + UpdateConfig(search::CompactionStrategy compactionStrategy) noexcept + : _compactionStrategy(compactionStrategy) + {} + void operator()(search::attribute::IAttributeVector &iAttributeVector) override { + auto attributeVector = dynamic_cast<search::AttributeVector *>(&iAttributeVector); + if (attributeVector != nullptr) { + auto cfg = attributeVector->getConfig(); + cfg.setCompactionStrategy(_compactionStrategy); + attributeVector->update_config(cfg); + } + } + search::CompactionStrategy _compactionStrategy; +}; + +} + +search::CompactionStrategy +StoreOnlyDocSubDB::computeCompactionStrategy(search::CompactionStrategy strategy) const { + return isNodeRetired() + ? search::CompactionStrategy(RETIRED_DEAD_RATIO, RETIRED_DEAD_RATIO) + : strategy; +} + void StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config, const AllocStrategy& alloc_strategy) { + _lastConfiguredCompactionStrategy = alloc_strategy.get_compaction_strategy(); auto cfg = _dms->getConfig(); GrowStrategy grow = alloc_strategy.get_grow_strategy(); // Amortize memory spike cost over N docs grow.setDocsGrowDelta(grow.getDocsGrowDelta() + alloc_strategy.get_amortize_count()); cfg.setGrowStrategy(grow); - cfg.setCompactionStrategy(alloc_strategy.get_compaction_strategy()); + cfg.setCompactionStrategy(computeCompactionStrategy(alloc_strategy.get_compaction_strategy())); _dms->update_config(cfg); // Update grow and compaction config _rSummaryMgr->reconfigure(config); } void -StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) -{ +StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> & calc) { + bool wasNodeRetired = isNodeRetired(); + _nodeRetired = calc->nodeRetired(); + if (wasNodeRetired != isNodeRetired()) { + search::CompactionStrategy compactionStrategy = computeCompactionStrategy(_lastConfiguredCompactionStrategy); + auto cfg = _dms->getConfig(); + cfg.setCompactionStrategy(compactionStrategy); + _dms->update_config(cfg); + reconfigureAttributesConsideringNodeState(); + } +} + +void +StoreOnlyDocSubDB::reconfigureAttributesConsideringNodeState() { + search::CompactionStrategy compactionStrategy = computeCompactionStrategy(_lastConfiguredCompactionStrategy); + auto attrMan = getAttributeManager(); + if (attrMan) { + attrMan->asyncForEachAttribute(std::make_shared<UpdateConfig>(compactionStrategy)); + } } proton::IAttributeManager::SP diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index 50e5ffba0e2..7051722f605 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -152,6 +152,8 @@ private: DocumentMetaStoreFlushTarget::SP _dmsFlushTarget; std::shared_ptr<ShrinkLidSpaceFlushTarget> _dmsShrinkTarget; std::shared_ptr<PendingLidTrackerBase> _pendingLidsForCommit; + bool _nodeRetired; + search::CompactionStrategy _lastConfiguredCompactionStrategy; IFlushTargetList getFlushTargets() override; protected: @@ -180,9 +182,8 @@ protected: StoreOnlyFeedView::Context getStoreOnlyFeedViewContext(const DocumentDBConfig &configSnapshot); StoreOnlyFeedView::PersistentParams getFeedViewPersistentParams(); vespalib::string getSubDbName() const; - - void reconfigure(const search::LogDocumentStore::Config & protonConfig, - const AllocStrategy& alloc_strategy); + void reconfigure(const search::LogDocumentStore::Config & protonConfig, const AllocStrategy& alloc_strategy); + void reconfigureAttributesConsideringNodeState(); public: StoreOnlyDocSubDB(const Config &cfg, const Context &ctx); ~StoreOnlyDocSubDB() override; @@ -233,6 +234,9 @@ public: std::shared_ptr<IDocumentDBReference> getDocumentDBReference() override; void tearDownReferences(IDocumentDBReferenceResolver &resolver) override; PendingLidTrackerBase & getUncommittedLidsTracker() override { return *_pendingLidsForCommit; } + search::CompactionStrategy computeCompactionStrategy(search::CompactionStrategy strategy) const; + bool isNodeRetired() const { return _nodeRetired; } + }; } |