diff options
author | Geir Storli <geirst@yahooinc.com> | 2023-02-07 16:03:06 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-02-07 16:03:06 +0100 |
commit | 260788dbcc76f4b7b0855bfa48e23495a1561e91 (patch) | |
tree | 801c3b04edadaf93f3c05958031202f9adb39668 /searchcore | |
parent | c71bc7f0582f0c29476310dec2c1c12762bcbb8d (diff) | |
parent | 1fb56a093caf4369189e0d00ff3e406d674ec420 (diff) |
Merge pull request #25919 from vespa-engine/toregge/pass-optional-serial-num-to-prepare-reconfigv8.121.38
Pass optional serial num to prepare reconfig
Diffstat (limited to 'searchcore')
28 files changed, 116 insertions, 65 deletions
diff --git a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp index 29639448023..88d0752461b 100644 --- a/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp +++ b/searchcore/src/tests/proton/attribute/attribute_manager/attribute_manager_test.cpp @@ -214,6 +214,8 @@ struct Fixture : public BaseFixture, public AttributeManagerFixture struct SequentialAttributeManager { SequentialAttributesInitializer initializer; + uint32_t docid_limit; + SerialNum serial_num; proton::AttributeManager mgr; SequentialAttributeManager(const AttributeManager &currMgr, AttrMgrSpec && newSpec); ~SequentialAttributeManager(); @@ -221,9 +223,11 @@ struct SequentialAttributeManager SequentialAttributeManager::SequentialAttributeManager(const AttributeManager &currMgr, AttrMgrSpec && newSpec) : initializer(newSpec.getDocIdLimit()), + docid_limit(newSpec.getDocIdLimit()), + serial_num(newSpec.getCurrentSerialNum().value_or(0)), mgr(currMgr, std::move(newSpec), initializer) { - mgr.addInitializedAttributes(initializer.getInitializedAttributes(), std::nullopt, std::nullopt); + mgr.addInitializedAttributes(initializer.getInitializedAttributes(), docid_limit, serial_num); } SequentialAttributeManager::~SequentialAttributeManager() = default; diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 2877005e2c1..d7ce042a4d3 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -246,7 +246,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, IDocumentDBReferenceResolver& resolver, SerialNum serial_num) { - auto prepared_reconfig = _configurer->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); + auto prepared_reconfig = _configurer->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); _configurer->reconfigure(new_config_snapshot, old_config_snapshot, reconfig_params, resolver, *prepared_reconfig, serial_num); } @@ -258,7 +258,7 @@ Fixture::reconfigure(const DocumentDBConfig& new_config_snapshot, IDocumentDBReferenceResolver& resolver, SerialNum serial_num) { - auto prepared_reconfig = _configurer->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); + auto prepared_reconfig = _configurer->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); return _configurer->reconfigure(new_config_snapshot, old_config_snapshot, std::move(attr_spec), reconfig_params, resolver, *prepared_reconfig, serial_num); } @@ -341,7 +341,7 @@ FastAccessFixture::reconfigure(const DocumentDBConfig& new_config_snapshot, SerialNum serial_num) { ReconfigParams reconfig_params{CCR()}; - auto prepared_reconfig = _configurer.prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); + auto prepared_reconfig = _configurer.prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); return _configurer.reconfigure(new_config_snapshot, old_config_snapshot, std::move(attr_spec), *prepared_reconfig, serial_num); } diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 307695c4f4b..f49c46d81e8 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -379,7 +379,7 @@ struct FixtureBase cmpResult.documentTypeRepoChanged = true; MyDocumentDBReferenceResolver resolver; ReconfigParams reconfig_params(cmpResult); - auto prepared_reconfig = _subDb.prepare_reconfig(*newCfg->_cfg, *_snapshot->_cfg, reconfig_params); + auto prepared_reconfig = _subDb.prepare_reconfig(*newCfg->_cfg, *_snapshot->_cfg, reconfig_params, serialNum); auto tasks = _subDb.applyConfig(*newCfg->_cfg, *_snapshot->_cfg, serialNum, reconfig_params, resolver, *prepared_reconfig); prepared_reconfig.reset(); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp index c15a15e44e5..77cc1142f75 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_initializer.cpp @@ -86,7 +86,7 @@ void AttributeManagerInitializerTask::run() { _attrMgr->addExtraAttribute(_documentMetaStore); - _attrMgr->addInitializedAttributes(_attributesResult.get(), std::nullopt, std::nullopt); + _attrMgr->addInitializedAttributes(_attributesResult.get(), 1, _configSerialNum); _attrMgr->pruneRemovedFields(_configSerialNum); _promise.set_value(); } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.cpp index 2942256f534..06c03c80785 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.cpp @@ -17,7 +17,7 @@ AttributeManagerReconfig::AttributeManagerReconfig(std::shared_ptr<AttributeMana AttributeManagerReconfig::~AttributeManagerReconfig() = default; std::shared_ptr<AttributeManager> -AttributeManagerReconfig::create(std::optional<uint32_t> docid_limit, std::optional<search::SerialNum> serial_num) +AttributeManagerReconfig::create(uint32_t docid_limit, search::SerialNum serial_num) { assert(_mgr); _mgr->addInitializedAttributes(_initializer->getInitializedAttributes(), docid_limit, serial_num); diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.h b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.h index da8ae163fb5..9e4f8b71a7a 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attribute_manager_reconfig.h @@ -22,7 +22,7 @@ public: AttributeManagerReconfig(std::shared_ptr<AttributeManager> mgr, std::unique_ptr<SequentialAttributesInitializer> initializer); ~AttributeManagerReconfig(); - std::shared_ptr<AttributeManager> create(std::optional<uint32_t> docid_limit, std::optional<search::SerialNum> serial_num); + std::shared_ptr<AttributeManager> create(uint32_t docid_limit, search::SerialNum serial_num); }; } diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp index e27fae65de3..dd6cc1bdb85 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.cpp @@ -325,16 +325,24 @@ AttributeManager::addAttribute(AttributeSpec && spec, uint64_t serialNum) } void -AttributeManager::addInitializedAttributes(const std::vector<AttributeInitializerResult> &attributes, std::optional<uint32_t> docid_limit, std::optional<SerialNum> serial_num) +AttributeManager::addInitializedAttributes(const std::vector<AttributeInitializerResult> &attributes, uint32_t docid_limit, SerialNum serial_num) { + /* + * Called (indirectly) by + * DocumentSubDBCollection::complete_prepare_reconfig to complete + * setup of new attribute manager. + */ for (const auto &result : attributes) { assert(result); auto attr = result.getAttribute(); - if (docid_limit.has_value()) { - AttributeManager::padAttribute(*attr, docid_limit.value()); - } - if (serial_num.has_value()) { - attr->setCreateSerialNum(serial_num.value()); + if (attr->getCreateSerialNum() == 0) { + /* + * The attribute vector is empty (not loaded from disk) + * and has been added as part of live reconfig. Make it + * ready for use by setting size and create serial num. + */ + AttributeManager::padAttribute(*attr, docid_limit); + attr->setCreateSerialNum(serial_num); } attr->setInterlock(_interlock); auto shrinker = allocShrinker(attr, _attributeFieldWriter, *_diskLayout); @@ -503,8 +511,10 @@ proton::IAttributeManager::SP AttributeManager::create(Spec&& spec) const { assert(spec.getCurrentSerialNum().has_value()); + auto docid_limit = spec.getDocIdLimit(); + auto serial_num = spec.getCurrentSerialNum().value(); auto prepared_result = prepare_create(std::move(spec)); - return prepared_result->create(std::nullopt, std::nullopt); + return prepared_result->create(docid_limit, serial_num); } std::vector<IFlushTarget::SP> diff --git a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h index 35895e5422c..c8c5da7f4e4 100644 --- a/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h +++ b/searchcore/src/vespa/searchcore/proton/attribute/attributemanager.h @@ -123,7 +123,7 @@ public: AttributeVectorSP addAttribute(AttributeSpec && spec, uint64_t serialNum); - void addInitializedAttributes(const std::vector<AttributeInitializerResult> &attributes, std::optional<uint32_t> docid_limit, std::optional<SerialNum> serial_num); + void addInitializedAttributes(const std::vector<AttributeInitializerResult> &attributes, uint32_t docid_limit, SerialNum serial_num); void addExtraAttribute(const AttributeVectorSP &attribute); diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp index 903c9d0a899..f0d028f924f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.cpp @@ -5,8 +5,8 @@ namespace proton { -DocumentDBReconfig::DocumentDBReconfig(std::unique_ptr<const DocumentSubDBReconfig> ready_reconfig_in, - std::unique_ptr<const DocumentSubDBReconfig> not_ready_reconfig_in) +DocumentDBReconfig::DocumentDBReconfig(std::unique_ptr<DocumentSubDBReconfig> ready_reconfig_in, + std::unique_ptr<DocumentSubDBReconfig> not_ready_reconfig_in) : _ready_reconfig(std::move(ready_reconfig_in)), _not_ready_reconfig(std::move(not_ready_reconfig_in)) { diff --git a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h index 21ea117f7e9..d35a2211397 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_db_reconfig.h @@ -10,26 +10,31 @@ class DocumentSubDBReconfig; /** * Class representing the result of the prepare step of a DocumentDB reconfig. * - * The reconfig is performed in two steps: + * The reconfig is performed in three steps: * Prepare: * Based on the config that is changed, new components are instantiated in each subdb. * This can be costly and is handled by helper threads from the shared executor pool. * + * Complete prepare: + * Docid limit and serial number are used to complete prepared reconfig. + * * Apply: * The new components are swapped with the old ones in the DocumentDB master write thread. */ class DocumentDBReconfig { private: - std::unique_ptr<const DocumentSubDBReconfig> _ready_reconfig; - std::unique_ptr<const DocumentSubDBReconfig> _not_ready_reconfig; + std::unique_ptr<DocumentSubDBReconfig> _ready_reconfig; + std::unique_ptr<DocumentSubDBReconfig> _not_ready_reconfig; public: - DocumentDBReconfig(std::unique_ptr<const DocumentSubDBReconfig> ready_reconfig_in, - std::unique_ptr<const DocumentSubDBReconfig> not_ready_reconfig_in); + DocumentDBReconfig(std::unique_ptr<DocumentSubDBReconfig> ready_reconfig_in, + std::unique_ptr<DocumentSubDBReconfig> not_ready_reconfig_in); ~DocumentDBReconfig(); - const DocumentSubDBReconfig& ready_reconfig() const { return *_ready_reconfig; } - const DocumentSubDBReconfig& not_ready_reconfig() const { return *_not_ready_reconfig; } + const DocumentSubDBReconfig& ready_reconfig() const noexcept { return *_ready_reconfig; } + DocumentSubDBReconfig& ready_reconfig() noexcept { return *_ready_reconfig; } + const DocumentSubDBReconfig& not_ready_reconfig() const noexcept { return *_not_ready_reconfig; } + DocumentSubDBReconfig& not_ready_reconfig() noexcept { return *_not_ready_reconfig; } }; } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp index 2fd222338e7..60514b91aab 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.cpp @@ -12,5 +12,12 @@ DocumentSubDBReconfig::DocumentSubDBReconfig(std::shared_ptr<Matchers> matchers_ DocumentSubDBReconfig::~DocumentSubDBReconfig() = default; +void +DocumentSubDBReconfig::complete(uint32_t docid_limit, search::SerialNum serial_num) +{ + (void) docid_limit; + (void) serial_num; +} + } diff --git a/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h index 89129a6b0d0..0554056100e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/document_subdb_reconfig.h @@ -1,6 +1,7 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #pragma once +#include <vespa/searchlib/common/serialnum.h> #include <memory> namespace proton { @@ -26,7 +27,8 @@ public: return _old_matchers != _new_matchers; } std::shared_ptr<Matchers> matchers() const { return _new_matchers; } + + void complete(uint32_t docid_limit, search::SerialNum serial_num); }; } - diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 110b0324dc3..a217e3b719e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -349,7 +349,7 @@ DocumentDB::initFinish(DocumentDBConfig::SP configSnapshot) } std::unique_ptr<DocumentDBReconfig> -DocumentDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot) +DocumentDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, std::optional<SerialNum> serial_num) { auto active_config_snapshot = getActiveConfig(); auto cmpres = active_config_snapshot->compare(new_config_snapshot); @@ -357,7 +357,7 @@ DocumentDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot) cmpres.importedFieldsChanged = true; } const ReconfigParams reconfig_params(cmpres); - return _subDBs.prepare_reconfig(new_config_snapshot, *active_config_snapshot, reconfig_params); + return _subDBs.prepare_reconfig(new_config_snapshot, *active_config_snapshot, reconfig_params, serial_num); } void @@ -423,7 +423,7 @@ DocumentDB::applySubDBConfig(const DocumentDBConfig &newConfigSnapshot, } void -DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum, std::unique_ptr<const DocumentDBReconfig> prepared_reconfig) +DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum, std::unique_ptr<DocumentDBReconfig> prepared_reconfig) { // Always called by executor thread: // Called by performReconfig() by executor thread during normal @@ -472,6 +472,7 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum bool elidedConfigSave = equalReplayConfig && tlsReplayDone; forceCommitAndWait(*_feedView.get(), elidedConfigSave ? serialNum : serialNum - 1, std::move(commit_result)); } + _subDBs.complete_prepare_reconfig(*prepared_reconfig, serialNum); if (params.shouldMaintenanceControllerChange()) { _maintenanceController.killJobs(); } @@ -796,7 +797,7 @@ DocumentDB::reconfigure(DocumentDBConfig::SP snapshot) assert(active_snapshot); assert(_state.getAllowReconfig()); snapshot = DocumentDBConfig::makeDelayedAttributeAspectConfig(snapshot, *active_snapshot); - auto prepared_reconfig = prepare_reconfig(*snapshot); + auto prepared_reconfig = prepare_reconfig(*snapshot, std::nullopt); masterExecute([this, snapshot, prepared_reconfig = std::move(prepared_reconfig)]() mutable { performReconfig(snapshot, std::move(prepared_reconfig)); }); // Wait for config to be applied, or for document db close std::unique_lock<std::mutex> guard(_configMutex); @@ -835,7 +836,7 @@ DocumentDB::enterApplyLiveConfigState() (void) _state.enterApplyLiveConfigState(); } auto new_config_snapshot = _pendingConfigSnapshot.get(); - auto prepared_reconfig = prepare_reconfig(*new_config_snapshot); + auto prepared_reconfig = prepare_reconfig(*new_config_snapshot, std::nullopt); masterExecute([this, new_config_snapshot, prepared_reconfig = std::move(prepared_reconfig)]() mutable { performReconfig(std::move(new_config_snapshot), std::move(prepared_reconfig)); @@ -896,7 +897,7 @@ DocumentDB::replayConfig(search::SerialNum serialNum) configSnapshot = DocumentDBConfigScout::scout(configSnapshot, *_pendingConfigSnapshot.get()); // Ignore configs that are not relevant during replay of transaction log configSnapshot = DocumentDBConfig::makeReplayConfig(configSnapshot); - auto prepared_reconfig = prepare_reconfig(*configSnapshot); + auto prepared_reconfig = prepare_reconfig(*configSnapshot, serialNum); applyConfig(configSnapshot, serialNum, std::move(prepared_reconfig)); LOG(info, "DocumentDB(%s): Replayed config with serialNum=%" PRIu64, _docTypeName.toString().c_str(), serialNum); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 2b3e6ef4052..6e611917a79 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -156,7 +156,7 @@ private: void applySubDBConfig(const DocumentDBConfig &newConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, const DocumentDBReconfig& prepared_reconfig); - void applyConfig(DocumentDBConfigSP configSnapshot, SerialNum serialNum, std::unique_ptr<const DocumentDBReconfig> prepared_reconfig); + void applyConfig(DocumentDBConfigSP configSnapshot, SerialNum serialNum, std::unique_ptr<DocumentDBReconfig> prepared_reconfig); /** * Save initial config if we don't have any saved config snapshots. @@ -382,7 +382,7 @@ public: bool getDelayedConfig() const { return _state.getDelayedConfig(); } void replayConfig(SerialNum serialNum) override; const DocTypeName & getDocTypeName() const { return _docTypeName; } - std::unique_ptr<DocumentDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot); + std::unique_ptr<DocumentDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, std::optional<SerialNum> serial_num); void reconfigure(DocumentDBConfigSP snapshot) override; int64_t getActiveGeneration() const; /* diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp index 573683c21a9..73f90df11ab 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.cpp @@ -251,14 +251,21 @@ DocumentSubDBCollection::pruneRemovedFields(SerialNum serialNum) } std::unique_ptr<DocumentDBReconfig> -DocumentSubDBCollection::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) +DocumentSubDBCollection::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) { - auto ready_reconfig = getReadySubDB()->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); - auto not_ready_reconfig = getNotReadySubDB()->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); + auto ready_reconfig = getReadySubDB()->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); + auto not_ready_reconfig = getNotReadySubDB()->prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); return std::make_unique<DocumentDBReconfig>(std::move(ready_reconfig), std::move(not_ready_reconfig)); } void +DocumentSubDBCollection::complete_prepare_reconfig(DocumentDBReconfig& prepared_reconfig, SerialNum serial_num) +{ + getReadySubDB()->complete_prepare_reconfig(prepared_reconfig.ready_reconfig(), serial_num); + getNotReadySubDB()->complete_prepare_reconfig(prepared_reconfig.not_ready_reconfig(), serial_num); +} + +void DocumentSubDBCollection::applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, @@ -271,7 +278,7 @@ DocumentSubDBCollection::applyConfig(const DocumentDBConfig &newConfigSnapshot, _reprocessingRunner.addTasks(tasks); tasks = getNotReadySubDB()->applyConfig(newConfigSnapshot, oldConfigSnapshot, serialNum, params, resolver, prepared_reconfig.not_ready_reconfig()); _reprocessingRunner.addTasks(tasks); - auto removed_reconfig = getRemSubDB()->prepare_reconfig(newConfigSnapshot, oldConfigSnapshot, params); + auto removed_reconfig = getRemSubDB()->prepare_reconfig(newConfigSnapshot, oldConfigSnapshot, params, serialNum); tasks = getRemSubDB()->applyConfig(newConfigSnapshot, oldConfigSnapshot, serialNum, params, resolver, *removed_reconfig); removed_reconfig.reset(); _reprocessingRunner.addTasks(tasks); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h index 21f36c7ca3e..a359ee8f0f3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentsubdbcollection.h @@ -9,6 +9,7 @@ #include <vespa/vespalib/util/varholder.h> #include <vespa/vespalib/util/idestructorcallback.h> #include <mutex> +#include <optional> namespace vespalib { class Clock; @@ -137,7 +138,8 @@ public: void pruneRemovedFields(SerialNum serialNum); - std::unique_ptr<DocumentDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params); + std::unique_ptr<DocumentDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num); + void complete_prepare_reconfig(DocumentDBReconfig& prepared_reconfig, SerialNum serial_num); void applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver, const DocumentDBReconfig& prepared_reconfig); diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp index 5a2e9447f9c..eef605cea3c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.cpp @@ -243,10 +243,10 @@ FastAccessDocSubDB::initViews(const DocumentDBConfig &configSnapshot) } } -std::unique_ptr<const DocumentSubDBReconfig> -FastAccessDocSubDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) +std::unique_ptr<DocumentSubDBReconfig> +FastAccessDocSubDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) { - return _configurer.prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); + return _configurer.prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); } IReprocessingTask::List diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h index cec2e8595cf..e4c140e43e6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb.h @@ -110,8 +110,8 @@ public: void setup(const DocumentSubDbInitializerResult &initResult) override; void initViews(const DocumentDBConfig &configSnapshot) override; - std::unique_ptr<const DocumentSubDBReconfig> - prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) override; + std::unique_ptr<DocumentSubDBReconfig> + prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) override; IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver, const DocumentSubDBReconfig& prepared_reconfig) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp index a8211d246e1..0a88b591f9f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.cpp @@ -45,13 +45,14 @@ FastAccessDocSubDBConfigurer::FastAccessDocSubDBConfigurer(FeedViewVarHolder &fe FastAccessDocSubDBConfigurer::~FastAccessDocSubDBConfigurer() = default; -std::unique_ptr<const DocumentSubDBReconfig> -FastAccessDocSubDBConfigurer::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) +std::unique_ptr<DocumentSubDBReconfig> +FastAccessDocSubDBConfigurer::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<search::SerialNum> serial_num) { (void) new_config_snapshot; (void) old_config_snapshot; (void) reconfig_params; - return std::make_unique<const DocumentSubDBReconfig>(std::shared_ptr<Matchers>()); + (void) serial_num; + return std::make_unique<DocumentSubDBReconfig>(std::shared_ptr<Matchers>()); } IReprocessingInitializer::UP diff --git a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h index 6d57a9cc3de..7a50aa35c81 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/fast_access_doc_subdb_configurer.h @@ -33,7 +33,7 @@ public: const vespalib::string &subDbName); ~FastAccessDocSubDBConfigurer(); - std::unique_ptr<const DocumentSubDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params); + std::unique_ptr<DocumentSubDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<search::SerialNum> serial_num); IReprocessingInitializer::UP reconfigure(const DocumentDBConfig &newConfig, const DocumentDBConfig &oldConfig, diff --git a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h index 99804fe253f..bdf3d7c7102 100644 --- a/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/idocumentsubdb.h @@ -7,6 +7,7 @@ #include <vespa/searchlib/util/searchable_stats.h> #include <vespa/vespalib/stllike/string.h> #include <vespa/vespalib/util/idestructorcallback.h> +#include <optional> namespace search::index { class Schema; } @@ -79,8 +80,9 @@ public: virtual void setup(const DocumentSubDbInitializerResult &initResult) = 0; virtual void initViews(const DocumentDBConfig &configSnapshot) = 0; - virtual std::unique_ptr<const DocumentSubDBReconfig> - prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) = 0; + virtual std::unique_ptr<DocumentSubDBReconfig> + prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) = 0; + virtual void complete_prepare_reconfig(DocumentSubDBReconfig& prepared_reconfig, SerialNum serial_num) = 0; virtual IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver, const DocumentSubDBReconfig& prepared_reconfig) = 0; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index a934a445470..fa4340f7659 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -140,10 +140,11 @@ SearchableDocSubDBConfigurer::reconfigureIndexSearchable() reconfigureMatchView(indexManager->getSearchable()); } -std::unique_ptr<const DocumentSubDBReconfig> -SearchableDocSubDBConfigurer::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) +std::unique_ptr<DocumentSubDBReconfig> +SearchableDocSubDBConfigurer::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<search::SerialNum> serial_num) { (void) old_config_snapshot; + (void) serial_num; auto old_matchers = _searchView.get()->getMatchers(); auto reconfig = std::make_unique<DocumentSubDBReconfig>(std::move(old_matchers)); if (reconfig_params.shouldMatchersChange()) { diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index b3ec169afeb..e6b0ebda7c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -78,7 +78,7 @@ public: void reconfigureIndexSearchable(); - std::unique_ptr<const DocumentSubDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params); + std::unique_ptr<DocumentSubDBReconfig> prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<search::SerialNum> serial_num); void reconfigure(const DocumentDBConfig &newConfig, const DocumentDBConfig &oldConfig, diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 89925e8a77d..187be079d03 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -132,10 +132,10 @@ reconfigureMatchingMetrics(const RankProfilesConfig &cfg) } } -std::unique_ptr<const DocumentSubDBReconfig> -SearchableDocSubDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) +std::unique_ptr<DocumentSubDBReconfig> +SearchableDocSubDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) { - return _configurer.prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params); + return _configurer.prepare_reconfig(new_config_snapshot, old_config_snapshot, reconfig_params, serial_num); } IReprocessingTask::List diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 63100f7e6a9..522ccd69b2a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -100,8 +100,8 @@ public: void setup(const DocumentSubDbInitializerResult &initResult) override; void initViews(const DocumentDBConfig &configSnapshot) override; - std::unique_ptr<const DocumentSubDBReconfig> - prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) override; + std::unique_ptr<DocumentSubDBReconfig> + prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) override; IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver, const DocumentSubDBReconfig& prepared_reconfig) override; diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index 22aeeec4ef6..e3cf394c143 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -405,13 +405,20 @@ StoreOnlyDocSubDB::getSubDbName() const { return vespalib::make_string("%s.%s", _owner.getName().c_str(), _subName.c_str()); } -std::unique_ptr<const DocumentSubDBReconfig> -StoreOnlyDocSubDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) +std::unique_ptr<DocumentSubDBReconfig> +StoreOnlyDocSubDB::prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) { (void) new_config_snapshot; (void) old_config_snapshot; (void) reconfig_params; - return std::make_unique<const DocumentSubDBReconfig>(std::shared_ptr<Matchers>()); + (void) serial_num; + return std::make_unique<DocumentSubDBReconfig>(std::shared_ptr<Matchers>()); +} + +void +StoreOnlyDocSubDB::complete_prepare_reconfig(DocumentSubDBReconfig& prepared_reconfig, SerialNum serial_num) +{ + prepared_reconfig.complete(_dms->getCommittedDocIdLimit(), serial_num); } IReprocessingTask::List diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h index f547ced67c7..0abb06a76aa 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.h @@ -200,8 +200,9 @@ public: void validateDocStore(FeedHandler & feedHandler, SerialNum serialNum) const override; - std::unique_ptr<const DocumentSubDBReconfig> - prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params) override; + std::unique_ptr<DocumentSubDBReconfig> + prepare_reconfig(const DocumentDBConfig& new_config_snapshot, const DocumentDBConfig& old_config_snapshot, const ReconfigParams& reconfig_params, std::optional<SerialNum> serial_num) override; + void complete_prepare_reconfig(DocumentSubDBReconfig& prepared_reconfig, SerialNum serial_num) override; IReprocessingTask::List applyConfig(const DocumentDBConfig &newConfigSnapshot, const DocumentDBConfig &oldConfigSnapshot, SerialNum serialNum, const ReconfigParams ¶ms, IDocumentDBReferenceResolver &resolver, const DocumentSubDBReconfig &prepared_reconfig) override; diff --git a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h index 85315158c3c..5ab0109578c 100644 --- a/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h +++ b/searchcore/src/vespa/searchcore/proton/test/dummy_document_sub_db.h @@ -52,9 +52,10 @@ struct DummyDocumentSubDb : public IDocumentSubDB } void setup(const DocumentSubDbInitializerResult &) override {} void initViews(const DocumentDBConfig &) override {} - std::unique_ptr<const DocumentSubDBReconfig> prepare_reconfig(const DocumentDBConfig&, const DocumentDBConfig&, const ReconfigParams&) override { - return std::make_unique<const DocumentSubDBReconfig>(std::shared_ptr<Matchers>()); + std::unique_ptr<DocumentSubDBReconfig> prepare_reconfig(const DocumentDBConfig&, const DocumentDBConfig&, const ReconfigParams&, std::optional<SerialNum>) override { + return std::make_unique<DocumentSubDBReconfig>(std::shared_ptr<Matchers>()); } + void complete_prepare_reconfig(DocumentSubDBReconfig&, SerialNum) override { } IReprocessingTask::List applyConfig(const DocumentDBConfig &, const DocumentDBConfig &, SerialNum, const ReconfigParams &, IDocumentDBReferenceResolver &, const DocumentSubDBReconfig&) override { |