From ac0e328d1a66a082b122be79055a5b7815743271 Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Tue, 20 Sep 2022 13:25:04 +0200 Subject: Only save config needed for replay of transaction log. --- .../src/vespa/searchcore/proton/server/documentdb.cpp | 13 ++++++++----- searchcore/src/vespa/searchcore/proton/server/documentdb.h | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 1df2f405e02..64b04967f3f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -233,7 +233,7 @@ DocumentDB::DocumentDB(const vespalib::string &baseDir, _feedHandler->init(_config_store->getOldestSerialNum()); _feedHandler->setBucketDBHandler(&_subDBs.getBucketDBHandler()); - saveInitialConfig(*configSnapshot); + saveInitialConfig(configSnapshot); resumeSaveConfig(); SerialNum configSerial = _config_store->getPrevValidSerial(_feedHandler->getPrunedSerialNum() + 1); assert(configSerial > 0); @@ -473,15 +473,17 @@ DocumentDB::applyConfig(DocumentDBConfig::SP configSnapshot, SerialNum serialNum const ReconfigParams params(cmpres); // Save config via config manager if replay is done. + auto replay_config = DocumentDBConfig::makeReplayConfig(configSnapshot); bool equalReplayConfig = - *DocumentDBConfig::makeReplayConfig(configSnapshot) == + *replay_config == *DocumentDBConfig::makeReplayConfig(_activeConfigSnapshot); bool tlsReplayDone = _feedHandler->getTransactionLogReplayDone(); FeedHandler::CommitResult commit_result; if (!equalReplayConfig && tlsReplayDone) { sync(_feedHandler->getSerialNum()); serialNum = _feedHandler->inc_serial_num(); - _config_store->saveConfig(*configSnapshot, serialNum); + _config_store->saveConfig(*replay_config, serialNum); + replay_config.reset(); // save entry in transaction log NewConfigOperation op(serialNum, *_config_store); commit_result = _feedHandler->storeOperationSync(op); @@ -632,7 +634,7 @@ DocumentDB::getNumActiveDocs() const } void -DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) +DocumentDB::saveInitialConfig(std::shared_ptr configSnapshot) { // Only called from ctor @@ -657,7 +659,8 @@ DocumentDB::saveInitialConfig(const DocumentDBConfig &configSnapshot) LOG(warning, "DocumentDB(%s): saveInitialConfig() failed pruning due to '%s'", _docTypeName.toString().c_str(), e.what()); } - _config_store->saveConfig(configSnapshot, confSerial); + auto replay_config = DocumentDBConfig::makeReplayConfig(configSnapshot); + _config_store->saveConfig(*replay_config, confSerial); } void diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.h b/searchcore/src/vespa/searchcore/proton/server/documentdb.h index 9a37af71bab..ef885113079 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.h @@ -155,7 +155,7 @@ private: * * @param configSnapshot initial config snapshot. */ - void saveInitialConfig(const DocumentDBConfig &configSnapshot); + void saveInitialConfig(std::shared_ptr configSnapshot); /** * Resume interrupted config save if needed. -- cgit v1.2.3 From 983691d0ec9f3e1f9b719a29d468a8675f62962e Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Tue, 20 Sep 2022 17:57:28 +0200 Subject: Make replay schema based on replay summary config. --- .../src/apps/tests/persistenceconformance_test.cpp | 7 +------ .../documentdbconfig/documentdbconfig_test.cpp | 13 ++++++++---- .../proton_configurer/proton_configurer_test.cpp | 8 +------- .../src/vespa/searchcore/bmcluster/bm_node.cpp | 7 +------ .../searchcore/proton/server/documentdbconfig.cpp | 24 +++++++++++++++++++--- .../searchcore/proton/server/documentdbconfig.h | 5 +++++ .../proton/server/documentdbconfigmanager.cpp | 20 ++---------------- 7 files changed, 40 insertions(+), 44 deletions(-) diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 4c3acb403e8..a40ebdfa6ff 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -16,7 +16,6 @@ #include #include #include -#include #include #include #include @@ -61,7 +60,6 @@ using document::test::makeBucketSpace; using search::TuneFileDocumentDB; using search::index::DummyFileHeaderContext; using search::index::Schema; -using search::index::SchemaBuilder; using search::transactionlog::TransLogServer; using storage::spi::ConformanceTest; using storage::spi::PersistenceProvider; @@ -126,10 +124,7 @@ public: CS::IndexschemaConfigSP indexschema = _schemaFactory->createIndexSchema(*docType); CS::AttributesConfigSP attributes = _schemaFactory->createAttributes(*docType); CS::SummaryConfigSP summary = _schemaFactory->createSummary(*docType); - Schema::SP schema(new Schema()); - SchemaBuilder::build(*indexschema, *schema); - SchemaBuilder::build(*attributes, *schema); - SchemaBuilder::build(*summary, *schema); + auto schema = DocumentDBConfig::build_schema(*attributes, *summary, *indexschema); return std::make_shared( 1, std::make_shared(), diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index c2d3da1b4f6..f1d6a384e9c 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -123,22 +123,27 @@ public: }; struct Fixture { - Schema::SP schema; + std::shared_ptr basic_schema; + std::shared_ptr full_schema; std::shared_ptr repo; ConfigSP basicCfg; ConfigSP fullCfg; ConfigSP replayCfg; ConfigSP nullCfg; Fixture() - : schema(make_shared()), + : basic_schema(make_shared()), + full_schema(make_shared()), repo(make_shared()), basicCfg(), fullCfg(), replayCfg(), nullCfg() { - basicCfg = MyConfigBuilder(4, schema, repo).addAttribute().addSummary(false, false).build(); - fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). + basic_schema->addAttributeField(Schema::AttributeField("my_attribute", schema::DataType::INT32)); + full_schema->addAttributeField(Schema::AttributeField("my_attribute", schema::DataType::INT32)); + full_schema->addSummaryField(Schema::SummaryField("my_attribute", schema::DataType::INT32)); + basicCfg = MyConfigBuilder(4, basic_schema, repo).addAttribute().addSummary(false, false).build(); + fullCfg = MyConfigBuilder(4, full_schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). addRankingExpression(). diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 48ff1dac93a..b9278c41d1d 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -20,7 +20,6 @@ #include #include #include -#include #include #include #include @@ -42,7 +41,6 @@ using document::DocumentTypeRepo; using search::TuneFileDocumentDB; using std::map; using search::index::Schema; -using search::index::SchemaBuilder; using proton::matching::RankingConstants; using proton::matching::RankingExpressions; using proton::matching::OnnxModels; @@ -58,11 +56,7 @@ struct DBConfigFixture { Schema::SP buildSchema() { - Schema::SP schema(std::make_shared()); - SchemaBuilder::build(_attributesBuilder, *schema); - SchemaBuilder::build(_summaryBuilder, *schema); - SchemaBuilder::build(_indexschemaBuilder, *schema); - return schema; + return DocumentDBConfig::build_schema(_attributesBuilder, _summaryBuilder, _indexschemaBuilder); } static RankingConstants::SP buildRankingConstants() diff --git a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp index 2af8f909dac..cfcb7af2472 100644 --- a/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp +++ b/searchcore/src/vespa/searchcore/bmcluster/bm_node.cpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -91,7 +90,6 @@ using proton::DocumentDB; using proton::DocumentDBConfig; using proton::HwInfo; using search::index::Schema; -using search::index::SchemaBuilder; using search::transactionlog::TransLogServer; using storage::MergeThrottler; using storage::distributor::BucketSpacesStatsProvider; @@ -178,10 +176,7 @@ std::shared_ptr make_document_db_config(std::shared_ptr(); auto attributes = make_attributes_config(); auto summary = std::make_shared(); - std::shared_ptr schema(new Schema()); - SchemaBuilder::build(*indexschema, *schema); - SchemaBuilder::build(*attributes, *schema); - SchemaBuilder::build(*summary, *schema); + auto schema = DocumentDBConfig::build_schema(*attributes, *summary, *indexschema); return std::make_shared( 1, std::make_shared(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index 70478be5b70..5992762b323 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ using namespace vespa::config::search; using document::DocumentTypeRepo; using search::TuneFileDocumentDB; using search::index::Schema; +using search::index::SchemaBuilder; using vespa::config::search::core::RankingConstantsConfig; using vespa::config::search::core::OnnxModelsConfig; @@ -233,7 +235,12 @@ DocumentDBConfig::SP DocumentDBConfig::makeReplayConfig(const SP & orig) { const DocumentDBConfig &o = *orig; - + + auto replay_summary_config = emptyConfig(o._summary); + auto replay_schema = build_schema(*o._attributes, *replay_summary_config, *o._indexschema); + if (*replay_schema == *o._schema) { + replay_schema = o._schema; + } SP ret = std::make_shared( o._generation, emptyConfig(o._rankProfiles), @@ -242,13 +249,13 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) std::make_shared(), o._indexschema, o._attributes, - emptyConfig(o._summary), + replay_summary_config, o._juniperrc, o._documenttypes, o._repo, std::make_shared(), o._tuneFileDocumentDB, - o._schema, + replay_schema, o._maintenance, o._storeConfig, o._threading_service_config, @@ -347,5 +354,16 @@ DocumentDBConfig::getDocumentType() const return _repo->getDocumentType(getDocTypeName()); } +std::shared_ptr +DocumentDBConfig::build_schema(const AttributesConfig& attributes_config, + const SummaryConfig& summary_config, + const IndexschemaConfig &indexschema_config) +{ + auto schema = std::make_shared(); + SchemaBuilder::build(attributes_config, *schema); + SchemaBuilder::build(summary_config, *schema); + SchemaBuilder::build(indexschema_config, *schema); + return schema; +} } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 4c814de0b90..1d0980238a3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -250,6 +250,11 @@ public: * reprocessing. */ static SP makeDelayedAttributeAspectConfig(const SP &newCfg, const DocumentDBConfig &oldCfg); + + static std::shared_ptr + build_schema(const AttributesConfig& attributes_config, + const SummaryConfig& summary_config, + const IndexschemaConfig &indexschema_config); }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 8de142034d1..e24ca7a8793 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -72,22 +72,6 @@ DocumentDBConfigManager::createConfigKeySet() const return set; } -namespace { - -Schema::SP -buildNewSchema(const AttributesConfig &newAttributesConfig, - const SummaryConfig &newSummaryConfig, - const IndexschemaConfig &newIndexschemaConfig) -{ - Schema::SP schema = std::make_shared(); - SchemaBuilder::build(newAttributesConfig, *schema); - SchemaBuilder::build(newSummaryConfig, *schema); - SchemaBuilder::build(newIndexschemaConfig, *schema); - return schema; -} - -} - Schema::SP DocumentDBConfigManager::buildSchema(const AttributesConfig &newAttributesConfig, const SummaryConfig &newSummaryConfig, @@ -99,14 +83,14 @@ DocumentDBConfigManager::buildSchema(const AttributesConfig &newAttributesConfig oldSchema = _pendingConfigSnapshot->getSchemaSP(); } if (!oldSchema) { - return buildNewSchema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig); + return DocumentDBConfig::build_schema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig); } const DocumentDBConfig &old = *_pendingConfigSnapshot; if (old.getAttributesConfig() != newAttributesConfig || old.getSummaryConfig() != newSummaryConfig || old.getIndexschemaConfig() != newIndexschemaConfig) { - Schema::SP schema(buildNewSchema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig)); + auto schema = DocumentDBConfig::build_schema(newAttributesConfig, newSummaryConfig, newIndexschemaConfig); return (*oldSchema == *schema) ? oldSchema : schema; } return oldSchema; -- cgit v1.2.3