diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2017-12-07 11:50:24 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-12-07 11:50:24 +0100 |
commit | d47b69f22029077852de7663d51a24ca46a42108 (patch) | |
tree | 32f6ee378b247825ffc67dc1220e79be237dcace /searchcore | |
parent | ff0e8671bb8e4d58ae424cdce791bbe61eb3eb8b (diff) | |
parent | 8535cbb1cfde7c23d4947a9b1a1e062b0469c722 (diff) |
Merge pull request #4377 from vespa-engine/toregge/use-alternate-maxflushed-parameter-when-node-is-retired
Use alternate maxflushed parameter when node is retired.
Diffstat (limited to 'searchcore')
24 files changed, 211 insertions, 20 deletions
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 87efd74659d..d3567da3527 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -666,6 +666,42 @@ TEST("require that maintenance controller should change if some config has chang TEST_DO(assertMaintenanceControllerShouldChange(CCR().setMaintenanceChanged(true))); } +void +assertSubDbsShouldNotChange(DocumentDBConfig::ComparisonResult result) +{ + ReconfigParams params(result); + EXPECT_FALSE(params.configHasChanged()); + EXPECT_FALSE(params.shouldSubDbsChange()); +} + +void +assertSubDbsShouldChange(DocumentDBConfig::ComparisonResult result) +{ + ReconfigParams params(result); + EXPECT_TRUE(params.configHasChanged()); + EXPECT_TRUE(params.shouldSubDbsChange()); +} + + +TEST("require that subdbs should change if relevant config changed") +{ + TEST_DO(assertSubDbsShouldNotChange(CCR())); + EXPECT_FALSE(ReconfigParams(CCR().setMaintenanceChanged(true)).shouldSubDbsChange()); + TEST_DO(assertSubDbsShouldChange(CCR().setFlushChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setStoreChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setDocumenttypesChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setDocumentTypeRepoChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setSummaryChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setSummarymapChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setJuniperrcChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setAttributesChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setImportedFieldsChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setVisibilityDelayChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setRankProfilesChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true))); +} + TEST_MAIN() { TEST_RUN_ALL(); diff --git a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp index b84aa1c1c6c..0f69684373c 100644 --- a/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp +++ b/searchcore/src/tests/proton/documentdb/maintenancecontroller/maintenancecontroller_test.cpp @@ -497,7 +497,8 @@ public: _mcCfg->getLidSpaceCompactionConfig(), _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -514,7 +515,8 @@ public: _mcCfg->getLidSpaceCompactionConfig(), _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -531,7 +533,8 @@ public: _mcCfg->getLidSpaceCompactionConfig(), _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } @@ -546,7 +549,8 @@ public: cfg, _mcCfg->getAttributeUsageFilterConfig(), _mcCfg->getAttributeUsageSampleInterval(), - _mcCfg->getBlockableJobConfig())); + _mcCfg->getBlockableJobConfig(), + _mcCfg->getFlushConfig())); _mcCfg = newCfg; forwardMaintenanceConfig(); } diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index c6c810ae72f..f46f6c67fd4 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -143,9 +143,15 @@ index.warmup.time double default=0.0 restart # Indicate if we also want warm up with full unpack, instead of only cheaper seek. index.warmup.unpack bool default=false restart -## How many flushed indexes there can be befor fusion is forced. +## How many flushed indexes there can be befor fusion is forced while node is +## not in retired state. ## Setting to 1 will force an immediate fusion. -index.maxflushed int default=2 restart +index.maxflushed int default=2 + +## How many flushed indexes there can be befor fusion is forced while node is +## in retired state. +## Setting to 1 will force an immediate fusion. +index.maxflushedretired int default=20 ## How much memory is set aside for caching. ## Now only used for caching of dictionary lookups. diff --git a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h index 00dd4befed7..1607c19611a 100644 --- a/searchcore/src/vespa/searchcore/proton/index/indexmanager.h +++ b/searchcore/src/vespa/searchcore/proton/index/indexmanager.h @@ -115,6 +115,9 @@ public: virtual void setSchema(const Schema &schema, SerialNum serialNum) override { _maintainer.setSchema(schema, serialNum); } + virtual void setMaxFlushed(uint32_t maxFlushed) override { + _maintainer.setMaxFlushed(maxFlushed); + } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt index 508a0208e70..8ee6a4f73ae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/server/CMakeLists.txt @@ -14,6 +14,7 @@ vespa_add_library(searchcore_server STATIC disk_mem_usage_forwarder.cpp docstorevalidator.cpp document_db_explorer.cpp + document_db_flush_config.cpp document_db_maintenance_config.cpp document_meta_store_read_guards.cpp document_scan_iterator.cpp diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.cpp new file mode 100644 index 00000000000..d11b8ee397d --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.cpp @@ -0,0 +1,24 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include "document_db_maintenance_config.h" + +namespace proton { + +DocumentDBFlushConfig::DocumentDBFlushConfig() + : DocumentDBFlushConfig(2, 20) +{ +} + +DocumentDBFlushConfig::DocumentDBFlushConfig(uint32_t maxFlushed, uint32_t maxFlushedRetired) + : _maxFlushed(maxFlushed), + _maxFlushedRetired(maxFlushedRetired) +{ +} + +bool +DocumentDBFlushConfig::operator==(const DocumentDBFlushConfig &rhs) const +{ + return _maxFlushed == rhs._maxFlushed && + _maxFlushedRetired == rhs._maxFlushedRetired; +} + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.h new file mode 100644 index 00000000000..316d0716477 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_flush_config.h @@ -0,0 +1,21 @@ +// Copyright 2017 Yahoo Holdings. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#pragma once + +namespace proton { + +/* + * Flush config, used to adjust flush targets as needed. + */ +class DocumentDBFlushConfig { + uint32_t _maxFlushed; + uint32_t _maxFlushedRetired; + +public: + DocumentDBFlushConfig(); + DocumentDBFlushConfig(uint32_t maxFlushed, uint32_t maxFlushedRetired); + bool operator==(const DocumentDBFlushConfig &rhs) const; + uint32_t getMaxFlushed() const { return _maxFlushed; } + uint32_t getMaxFlushedRetired() const { return _maxFlushedRetired; } +}; + +} // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp index cc491230784..848b1f27574 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.cpp @@ -118,7 +118,8 @@ DocumentDBMaintenanceConfig::DocumentDBMaintenanceConfig() _lidSpaceCompaction(), _attributeUsageFilterConfig(), _attributeUsageSampleInterval(60.0), - _blockableJobConfig() + _blockableJobConfig(), + _flushConfig() { } @@ -131,7 +132,8 @@ DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig & const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction, const AttributeUsageFilterConfig &attributeUsageFilterConfig, double attributeUsageSampleInterval, - const BlockableMaintenanceJobConfig &blockableJobConfig) + const BlockableMaintenanceJobConfig &blockableJobConfig, + const DocumentDBFlushConfig &flushConfig) : _pruneRemovedDocuments(pruneRemovedDocuments), _heartBeat(heartBeat), _sessionCachePruneInterval(groupingSessionPruneInterval), @@ -139,7 +141,8 @@ DocumentDBMaintenanceConfig(const DocumentDBPruneRemovedDocumentsConfig & _lidSpaceCompaction(lidSpaceCompaction), _attributeUsageFilterConfig(attributeUsageFilterConfig), _attributeUsageSampleInterval(attributeUsageSampleInterval), - _blockableJobConfig(blockableJobConfig) + _blockableJobConfig(blockableJobConfig), + _flushConfig(flushConfig) { } @@ -154,7 +157,8 @@ operator==(const DocumentDBMaintenanceConfig &rhs) const _lidSpaceCompaction == rhs._lidSpaceCompaction && _attributeUsageFilterConfig == rhs._attributeUsageFilterConfig && _attributeUsageSampleInterval == rhs._attributeUsageSampleInterval && - _blockableJobConfig == rhs._blockableJobConfig; + _blockableJobConfig == rhs._blockableJobConfig && + _flushConfig == rhs._flushConfig; } } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h index 2d4249531ec..acbbc442c7a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_maintenance_config.h @@ -4,6 +4,7 @@ #include <vespa/vespalib/stllike/string.h> #include <vespa/searchcore/proton/attribute/attribute_usage_filter_config.h> #include <vespa/fastos/timestamp.h> +#include "document_db_flush_config.h" namespace proton { @@ -95,6 +96,7 @@ private: AttributeUsageFilterConfig _attributeUsageFilterConfig; double _attributeUsageSampleInterval; BlockableMaintenanceJobConfig _blockableJobConfig; + DocumentDBFlushConfig _flushConfig; public: DocumentDBMaintenanceConfig(); @@ -106,7 +108,8 @@ public: const DocumentDBLidSpaceCompactionConfig &lidSpaceCompaction, const AttributeUsageFilterConfig &attributeUsageFilterConfig, double attributeUsageSampleInterval, - const BlockableMaintenanceJobConfig &blockableJobConfig); + const BlockableMaintenanceJobConfig &blockableJobConfig, + const DocumentDBFlushConfig &flushConfig); bool operator==(const DocumentDBMaintenanceConfig &rhs) const; @@ -133,6 +136,7 @@ public: const BlockableMaintenanceJobConfig &getBlockableJobConfig() const { return _blockableJobConfig; } + const DocumentDBFlushConfig &getFlushConfig() const { return _flushConfig; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp index 2eee5dff09d..5693e56ccf2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.cpp @@ -12,7 +12,8 @@ DocumentSubDbInitializerResult::DocumentSubDbInitializerResult() _summaryManager(std::make_shared<SummaryManager::SP>()), _attributeManager(std::make_shared<AttributeManager::SP>()), _indexManager(std::make_shared<IIndexManager::SP>()), - _lidReuseDelayerConfig() + _lidReuseDelayerConfig(), + _flushConfig() { } @@ -23,5 +24,12 @@ setLidReuseDelayerConfig(LidReuseDelayerConfig lidReuseDelayerConfig_in) _lidReuseDelayerConfig = lidReuseDelayerConfig_in; } +void +DocumentSubDbInitializerResult:: +setFlushConfig(const DocumentDBFlushConfig &flushConfig) +{ + _flushConfig = flushConfig; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h index 67b524332eb..da628c1ab5d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_initializer_result.h @@ -7,6 +7,7 @@ #include <vespa/searchcore/proton/documentmetastore/document_meta_store_initializer_result.h> #include <vespa/searchcorespi/index/iindexmanager.h> #include <vespa/searchcore/proton/documentmetastore/lid_reuse_delayer_config.h> +#include "document_db_flush_config.h" namespace proton { @@ -24,6 +25,7 @@ private: std::shared_ptr<searchcorespi::IIndexManager::SP> _indexManager; using LidReuseDelayerConfig = documentmetastore::LidReuseDelayerConfig; LidReuseDelayerConfig _lidReuseDelayerConfig; + DocumentDBFlushConfig _flushConfig; public: DocumentSubDbInitializerResult(); @@ -56,6 +58,8 @@ public: const LidReuseDelayerConfig &lidReuseDelayerConfig() const { return _lidReuseDelayerConfig; } + void setFlushConfig(const DocumentDBFlushConfig &flushConfig); + const DocumentDBFlushConfig &getFlushConfig() const { return _flushConfig; } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index 44f4260f7f8..b83040b04ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -42,7 +42,8 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() schemaChanged(false), maintenanceChanged(false), storeChanged(false), - visibilityDelayChanged(false) + visibilityDelayChanged(false), + flushChanged(false) { } DocumentDBConfig::DocumentDBConfig( @@ -152,6 +153,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const retval.maintenanceChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get()); retval.storeChanged = (_storeConfig != rhs._storeConfig); retval.visibilityDelayChanged = (_maintenance->getVisibilityDelay() != rhs._maintenance->getVisibilityDelay()); + retval.flushChanged = !equals<DocumentDBMaintenanceConfig>(_maintenance.get(), rhs._maintenance.get(), [](const auto &l, const auto &r) { return l.getFlushConfig() == r.getFlushConfig(); }); return retval; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 4250ce61175..cfca754a3d6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -46,6 +46,7 @@ public: bool maintenanceChanged; bool storeChanged; bool visibilityDelayChanged; + bool flushChanged; ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } @@ -63,7 +64,20 @@ public: ComparisonResult &setMaintenanceChanged(bool val) { maintenanceChanged = val; return *this; } ComparisonResult &setStoreChanged(bool val) { storeChanged = val; return *this; } - ComparisonResult &setVisibilityDelayChanged(bool val) { visibilityDelayChanged = val; return *this; } + ComparisonResult &setVisibilityDelayChanged(bool val) { + visibilityDelayChanged = val; + if (val) { + maintenanceChanged = true; + } + return *this; + } + ComparisonResult &setFlushChanged(bool val) { + flushChanged = val; + if (val) { + maintenanceChanged = true; + } + return *this; + } }; using SP = std::shared_ptr<DocumentDBConfig>; @@ -117,6 +131,14 @@ private: } return rhs != NULL && *lhs == *rhs; } + template <typename T, typename Func> + bool equals(const T *lhs, const T *rhs, Func isEqual) const + { + if (lhs == NULL) { + return rhs == NULL; + } + return rhs != NULL && isEqual(*lhs, *rhs); + } public: DocumentDBConfig(int64_t generation, const RankProfilesConfigSP &rankProfiles, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 6aea5234fbe..b9f59c4c411 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -136,7 +136,10 @@ buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig, proton.writefilter.sampleinterval, BlockableMaintenanceJobConfig( proton.maintenancejobs.resourcelimitfactor, - proton.maintenancejobs.maxoutstandingmoveops)); + proton.maintenancejobs.maxoutstandingmoveops), + DocumentDBFlushConfig( + proton.index.maxflushed, + proton.index.maxflushedretired)); } template<typename T> diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index c766cc89bb3..4c1dd47569b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -309,6 +309,15 @@ DocumentSubDBCollection::close() } void +DocumentSubDBCollection::setBucketStateCalculator(const IBucketStateCalculatorSP &calc) +{ + _calc = calc; + for (auto subDb : _subDBs) { + subDb->setBucketStateCalculator(calc); + } +} + +void DocumentSubDBCollection::tearDownReferences(IDocumentDBReferenceResolver &resolver) { for (auto subDb : _subDBs) { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 7290250c59e..ea07e391f69 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -94,9 +94,7 @@ public: const HwInfo &hwInfo); ~DocumentSubDBCollection(); - void setBucketStateCalculator(const IBucketStateCalculatorSP &calc) { - _calc = calc; - } + void setBucketStateCalculator(const IBucketStateCalculatorSP &calc); void createRetrievers(); void maintenanceSync(MaintenanceController &mc, ICommitable &commit); diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 2251a24c58a..d41c2088518 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -25,6 +25,7 @@ class DocumentSubDbInitializerResult; class FeedHandler; class FileConfigManager; class IAttributeManager; +class IBucketStateCalculator; class IDcoumentRetriever; class IDocumentDBReferenceResolver; class IDocumentDBReference; @@ -72,6 +73,7 @@ public: virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) = 0; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) = 0; virtual std::shared_ptr<ISearchHandler> getSearchView() const = 0; virtual std::shared_ptr<IFeedView> getFeedView() const = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 377b012de99..8ec41ae3e3c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -64,7 +64,7 @@ bool ReconfigParams::shouldSubDbsChange() const { return shouldMatchersChange() || shouldAttributeManagerChange() || shouldSummaryManagerChange() - || _res.documentTypeRepoChanged || _res.documenttypesChanged || _res.storeChanged; + || _res.documentTypeRepoChanged || _res.documenttypesChanged || _res.storeChanged || _res.flushChanged; } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 4069a871210..e05ab4b798f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -5,6 +5,7 @@ #include "document_subdb_initializer.h" #include "reconfig_params.h" #include "i_document_subdb_owner.h" +#include "ibucketstatecalculator.h" #include <vespa/searchcore/proton/attribute/attribute_writer.h> #include <vespa/searchcore/proton/flushengine/threadedflushtarget.h> #include <vespa/searchcore/proton/index/index_manager_initializer.h> @@ -49,7 +50,9 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()), _numSearcherThreads(cfg._numSearcherThreads), _warmupExecutor(ctx._warmupExecutor), - _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()) + _realGidToLidChangeHandler(std::make_shared<GidToLidChangeHandler>()), + _flushConfig(), + _nodeRetired(false) { _gidToLidChangeHandler = _realGidToLidChangeHandler; } @@ -136,6 +139,7 @@ SearchableDocSubDB::setup(const DocumentSubDbInitializerResult &initResult) Parent::setup(initResult); setupIndexManager(initResult.indexManager()); _docIdLimit.set(_dms->getCommittedDocIdLimit()); + applyFlushConfig(initResult.getFlushConfig()); } void @@ -160,6 +164,7 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const StoreOnlyDocSubDB::reconfigure(newConfigSnapshot.getStoreConfig()); IReprocessingTask::List tasks; updateLidReuseDelayer(&newConfigSnapshot); + applyFlushConfig(newConfigSnapshot.getMaintenanceConfigSP()->getFlushConfig()); if (params.shouldMatchersChange() && _addMetrics) { reconfigureMatchingMetrics(newConfigSnapshot.getRankProfilesConfig()); } @@ -185,6 +190,27 @@ SearchableDocSubDB::applyConfig(const DocumentDBConfig &newConfigSnapshot, const } void +SearchableDocSubDB::applyFlushConfig(const DocumentDBFlushConfig &flushConfig) +{ + _flushConfig = flushConfig; + propagateFlushConfig(); +} + +void +SearchableDocSubDB::propagateFlushConfig() +{ + uint32_t maxFlushed = _nodeRetired ? _flushConfig.getMaxFlushedRetired() : _flushConfig.getMaxFlushed(); + _indexMgr->setMaxFlushed(maxFlushed); +} + +void +SearchableDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) +{ + _nodeRetired = calc->nodeRetired(); + propagateFlushConfig(); +} + +void SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const SessionManager::SP &sessionManager) { assert(_writeService.master().isCurrentThread()); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 6da1a337cda..0852e028c25 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -85,6 +85,8 @@ private: const size_t _numSearcherThreads; vespalib::ThreadExecutor &_warmupExecutor; std::shared_ptr<GidToLidChangeHandler> _realGidToLidChangeHandler; + DocumentDBFlushConfig _flushConfig; + bool _nodeRetired; // Note: lifetime of indexManager must be handled by caller. std::shared_ptr<initializer::InitializerTask> @@ -100,6 +102,8 @@ private: bool reconfigure(vespalib::Closure0<bool>::UP closure) override; void reconfigureIndexSearchable(); void syncViews(); + void applyFlushConfig(const DocumentDBFlushConfig &flushConfig); + void propagateFlushConfig(); protected: IFlushTargetList getFlushTargetsInternal() override; @@ -120,6 +124,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override; void clearViews() override; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index e78f8136d26..c5a8706a0ce 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -329,6 +329,7 @@ StoreOnlyDocSubDB::createInitializer(const DocumentDBConfig &configSnapshot, Ser LidReuseDelayerConfig lidReuseDelayerConfig(configSnapshot); result->writableResult().setLidReuseDelayerConfig(lidReuseDelayerConfig); + result->writableResult().setFlushConfig(configSnapshot.getMaintenanceConfigSP()->getFlushConfig()); return result; } @@ -454,6 +455,11 @@ StoreOnlyDocSubDB::reconfigure(const search::LogDocumentStore::Config & config) _rSummaryMgr->reconfigure(config); } +void +StoreOnlyDocSubDB::setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) +{ +} + proton::IAttributeManager::SP StoreOnlyDocSubDB::getAttributeManager() const { diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index f5038c252a7..d7b242f27ff 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -215,6 +215,7 @@ public: IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver) override; + virtual void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &calc) override; ISearchHandler::SP getSearchView() const override { return _iSearchView.get(); } IFeedView::SP getFeedView() const override { return _iFeedView.get(); } diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 6620795483d..66ae296566f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -55,6 +55,7 @@ struct DummyDocumentSubDb : public IDocumentSubDB { return IReprocessingTask::List(); } + void setBucketStateCalculator(const std::shared_ptr<IBucketStateCalculator> &) override { } ISearchHandler::SP getSearchView() const override { return ISearchHandler::SP(); } IFeedView::SP getFeedView() const override { return IFeedView::SP(); } void clearViews() override {} diff --git a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h index 9e26b1250ff..9e105ef8a7f 100644 --- a/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h +++ b/searchcore/src/vespa/searchcore/proton/test/mock_index_manager.h @@ -28,6 +28,7 @@ struct MockIndexManager : public searchcorespi::IIndexManager } virtual void setSchema(const Schema &, SerialNum) override {} virtual void heartBeat(SerialNum) override {} + virtual void setMaxFlushed(uint32_t) override { } }; } // namespace test |