diff options
author | Geir Storli <geirst@yahoo-inc.com> | 2017-02-15 13:47:21 +0000 |
---|---|---|
committer | Geir Storli <geirst@yahoo-inc.com> | 2017-02-15 13:47:21 +0000 |
commit | d7aed85e7b24038429593d82b5d4c9d3f5519984 (patch) | |
tree | 3f35851220231246661342c021e67f6113bde306 /searchcore | |
parent | f9b2604e9f3a95125ad4be569e6a6b444ead8021 (diff) |
Add imported fields config as part of DocumentDBConfig.
Diffstat (limited to 'searchcore')
8 files changed, 93 insertions, 54 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 8a0e3e4b78e..b8fc3def866 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -4,6 +4,7 @@ LOG_SETUP("persistenceconformance_test"); #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/config-imported-fields.h> #include <vespa/document/base/testdocman.h> #include <vespa/persistence/conformancetest/conformancetest.h> #include <vespa/searchcommon/common/schemaconfigurer.h> @@ -121,6 +122,7 @@ public: std::make_shared<JuniperrcConfig>(), _typeCfg, _repo, + std::make_shared<ImportedFieldsConfig>(), std::make_shared<TuneFileDocumentDB>(), schema, std::make_shared<DocumentDBMaintenanceConfig>(), diff --git a/searchcore/src/tests/proton/config/config.cpp b/searchcore/src/tests/proton/config/config.cpp index f280c609070..96ac9489a86 100644 --- a/searchcore/src/tests/proton/config/config.cpp +++ b/searchcore/src/tests/proton/config/config.cpp @@ -1,20 +1,21 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -#include <vespa/vespalib/testkit/testapp.h> -#include <vespa/vespalib/util/linkedptr.h> #include <map> -#include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> +#include <vespa/config-attributes.h> +#include <vespa/config-imported-fields.h> +#include <vespa/config-indexschema.h> +#include <vespa/config-rank-profiles.h> +#include <vespa/config-summary.h> +#include <vespa/config-summarymap.h> +#include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/searchcore/proton/server/bootstrapconfig.h> +#include <vespa/searchcore/proton/server/bootstrapconfigmanager.h> #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/protonconfigurer.h> -#include <vespa/vespalib/util/varholder.h> -#include <vespa/fileacquirer/config-filedistributorrpc.h> -#include <vespa/config-summarymap.h> -#include <vespa/config-summary.h> -#include <vespa/config-rank-profiles.h> -#include <vespa/config-attributes.h> -#include <vespa/config-indexschema.h> #include <vespa/searchsummary/config/config-juniperrc.h> +#include <vespa/vespalib/testkit/testapp.h> +#include <vespa/vespalib/util/linkedptr.h> +#include <vespa/vespalib/util/varholder.h> using namespace config; using namespace proton; @@ -41,6 +42,7 @@ struct DoctypeFixture { SummaryConfigBuilder summaryBuilder; SummarymapConfigBuilder summarymapBuilder; JuniperrcConfigBuilder juniperrcBuilder; + ImportedFieldsConfigBuilder importedFieldsBuilder; }; struct ConfigTestFixture { @@ -92,6 +94,7 @@ struct ConfigTestFixture { set.addBuilder(db.configid, &fixture->summaryBuilder); set.addBuilder(db.configid, &fixture->summarymapBuilder); set.addBuilder(db.configid, &fixture->juniperrcBuilder); + set.addBuilder(db.configid, &fixture->importedFieldsBuilder); dbConfig[name] = fixture; } @@ -208,7 +211,7 @@ TEST_FF("require_that_documentdb_config_manager_subscribes_for_config", DocumentDBConfigManager(f1.configId + "/typea", "typea")) { f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); - ASSERT_EQUAL(7u, keySet.size()); + ASSERT_EQUAL(8u, keySet.size()); ConfigRetriever retriever(keySet, f1.context); f2.forwardConfig(f1.getBootstrapConfig(1)); f2.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index adc2ee8a438..ce9fc592ff7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -1,11 +1,12 @@ // Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "documentdbconfig.h" -#include <vespa/config-summary.h> -#include <vespa/config-summarymap.h> -#include <vespa/config-rank-profiles.h> #include <vespa/config-attributes.h> +#include <vespa/config-imported-fields.h> #include <vespa/config-indexschema.h> +#include <vespa/config-rank-profiles.h> +#include <vespa/config-summary.h> +#include <vespa/config-summarymap.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/config/config-documenttypes.h> @@ -32,6 +33,7 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() juniperrcChanged(false), _documenttypesChanged(false), _documentTypeRepoChanged(false), + _importedFieldsChanged(false), _tuneFileDocumentDBChanged(false), _schemaChanged(false), _maintenanceChanged(false) @@ -48,6 +50,7 @@ DocumentDBConfig::DocumentDBConfig( const JuniperrcConfigSP &juniperrc, const DocumenttypesConfigSP &documenttypes, const DocumentTypeRepo::SP &repo, + const ImportedFieldsConfigSP &importedFields, const search::TuneFileDocumentDB::SP &tuneFileDocumentDB, const Schema::SP &schema, const DocumentDBMaintenanceConfig::SP &maintenance, @@ -66,6 +69,7 @@ DocumentDBConfig::DocumentDBConfig( _juniperrc(juniperrc), _documenttypes(documenttypes), _repo(repo), + _importedFields(importedFields), _tuneFileDocumentDB(tuneFileDocumentDB), _schema(schema), _maintenance(maintenance), @@ -88,6 +92,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _juniperrc(cfg._juniperrc), _documenttypes(cfg._documenttypes), _repo(cfg._repo), + _importedFields(cfg._importedFields), _tuneFileDocumentDB(cfg._tuneFileDocumentDB), _schema(cfg._schema), _maintenance(cfg._maintenance), @@ -117,6 +122,8 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()) && _repo.get() == rhs._repo.get() && + equals<ImportedFieldsConfig >(_importedFields.get(), + rhs._importedFields.get()) && equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()) && equals<Schema>(_schema.get(), @@ -148,6 +155,8 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const !equals<DocumenttypesConfig>(_documenttypes.get(), rhs._documenttypes.get()); retval._documentTypeRepoChanged = _repo.get() != rhs._repo.get(); + retval._importedFieldsChanged = + !equals<ImportedFieldsConfig >(_importedFields.get(), rhs._importedFields.get()); retval._tuneFileDocumentDBChanged = !equals<TuneFileDocumentDB>(_tuneFileDocumentDB.get(), rhs._tuneFileDocumentDB.get()); @@ -172,6 +181,7 @@ DocumentDBConfig::valid() const (_juniperrc.get() != NULL) && (_documenttypes.get() != NULL) && (_repo.get() != NULL) && + (_importedFields.get() != NULL) && (_tuneFileDocumentDB.get() != NULL) && (_schema.get() != NULL) && (_maintenance.get() != NULL); @@ -211,6 +221,7 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) o._juniperrc, o._documenttypes, o._repo, + std::make_shared<ImportedFieldsConfig>(), o._tuneFileDocumentDB, o._schema, o._maintenance, @@ -250,6 +261,7 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) _juniperrc, _documenttypes, _repo, + _importedFields, _tuneFileDocumentDB, _schema, _maintenance, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 9515f2ec85a..e37c0532740 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -20,6 +20,7 @@ namespace vespa { class InternalRankProfilesType; class InternalAttributesType; class InternalIndexschemaType; + class InternalImportedFieldsType; } namespace summary { namespace internal { class InternalJuniperrcType; } } } @@ -45,6 +46,7 @@ public: bool juniperrcChanged; bool _documenttypesChanged; bool _documentTypeRepoChanged; + bool _importedFieldsChanged; bool _tuneFileDocumentDBChanged; bool _schemaChanged; bool _maintenanceChanged; @@ -69,6 +71,8 @@ public: using DocumenttypesConfig = const document::internal::InternalDocumenttypesType; using DocumenttypesConfigSP = std::shared_ptr<DocumenttypesConfig>; using MaintenanceConfigSP = DocumentDBMaintenanceConfig::SP; + using ImportedFieldsConfig = const vespa::config::search::internal::InternalImportedFieldsType; + using ImportedFieldsConfigSP = std::shared_ptr<ImportedFieldsConfig>; private: vespalib::string _configId; @@ -83,6 +87,7 @@ private: JuniperrcConfigSP _juniperrc; DocumenttypesConfigSP _documenttypes; document::DocumentTypeRepo::SP _repo; + ImportedFieldsConfigSP _importedFields; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; search::index::Schema::SP _schema; MaintenanceConfigSP _maintenance; @@ -93,8 +98,9 @@ private: template <typename T> bool equals(const T * lhs, const T * rhs) const { - if (lhs == NULL) + if (lhs == NULL) { return rhs == NULL; + } return rhs != NULL && *lhs == *rhs; } public: @@ -108,17 +114,18 @@ public: const JuniperrcConfigSP &juniperrc, const DocumenttypesConfigSP &documenttypesConfig, const document::DocumentTypeRepo::SP &repo, - const search::TuneFileDocumentDB::SP & tuneFileDocumentDB, + const ImportedFieldsConfigSP &importedFields, + const search::TuneFileDocumentDB::SP &tuneFileDocumentDB, const search::index::Schema::SP &schema, const DocumentDBMaintenanceConfig::SP &maintenance, const vespalib::string &configId, const vespalib::string &docTypeName, - const config::ConfigSnapshot & extraConfig = config::ConfigSnapshot()); + const config::ConfigSnapshot &extraConfig = config::ConfigSnapshot()); DocumentDBConfig(const DocumentDBConfig &cfg); ~DocumentDBConfig(); - const vespalib::string & getConfigId() const { return _configId; } + const vespalib::string &getConfigId() const { return _configId; } void setConfigId(const vespalib::string &configId) { _configId = configId; } const vespalib::string &getDocTypeName() const { return _docTypeName; } @@ -143,6 +150,8 @@ public: const DocumenttypesConfigSP &getDocumenttypesConfigSP() const { return _documenttypes; } const document::DocumentTypeRepo::SP &getDocumentTypeRepoSP() const { return _repo; } const document::DocumentType *getDocumentType() const { return _repo->getDocumentType(getDocTypeName()); } + const ImportedFieldsConfig &getImportedFieldsConfig() const { return *_importedFields; } + const ImportedFieldsConfigSP &getImportedFieldsConfigSP() const { return _importedFields; } const search::index::Schema::SP &getSchemaSP() const { return _schema; } const MaintenanceConfigSP &getMaintenanceConfigSP() const { return _maintenance; } const search::TuneFileDocumentDB::SP &getTuneFileDocumentDBSP() const { return _tuneFileDocumentDB; } @@ -153,15 +162,15 @@ public: */ ComparisonResult compare(const DocumentDBConfig &rhs) const; - bool valid(void) const; + bool valid() const; - const config::ConfigSnapshot & getExtraConfigs() const { return _extraConfigs; } + const config::ConfigSnapshot &getExtraConfigs() const { return _extraConfigs; } void setExtraConfigs(const config::ConfigSnapshot &extraConfigs) { _extraConfigs = extraConfigs; } /** * Only keep configs needed for replay of transaction log. */ - static SP makeReplayConfig(const SP & orig); + static SP makeReplayConfig(const SP &orig); /** * Return original config if this is a replay config, otherwise return @@ -173,7 +182,7 @@ public: * Return original config if cfg is a replay config, otherwise return * cfg. */ - static SP preferOriginalConfig(const SP & cfg); + static SP preferOriginalConfig(const SP &cfg); /** * Create modified attributes config. diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index e0293036696..b749d7abd3b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -2,15 +2,16 @@ #include "documentdbconfigmanager.h" #include "bootstrapconfig.h" +#include <vespa/config-imported-fields.h> +#include <vespa/config-rank-profiles.h> +#include <vespa/config-summarymap.h> +#include <vespa/config/file_acquirer/file_acquirer.h> +#include <vespa/config/helper/legacy.h> +#include <vespa/log/log.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchlib/index/schemautil.h> -#include <vespa/config/helper/legacy.h> -#include <vespa/config/file_acquirer/file_acquirer.h> -#include <vespa/vespalib/time/time_box.h> -#include <vespa/config-summarymap.h> -#include <vespa/config-rank-profiles.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/log/log.h> +#include <vespa/vespalib/time/time_box.h> LOG_SETUP(".proton.server.documentdbconfigmanager"); @@ -38,7 +39,8 @@ DocumentDBConfigManager::createConfigKeySet() const AttributesConfig, SummaryConfig, SummarymapConfig, - JuniperrcConfig>(_configId); + JuniperrcConfig, + ImportedFieldsConfig>(_configId); set.add(_extraConfigKeys); return set; } @@ -129,7 +131,7 @@ buildMaintenanceConfig(const BootstrapConfig::SP &bootstrapConfig, void -DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) +DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) { using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr<vespa::config::search::core::RankingConstantsConfig>; @@ -138,6 +140,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) using SummaryConfigSP = DocumentDBConfig::SummaryConfigSP; using SummarymapConfigSP = DocumentDBConfig::SummarymapConfigSP; using JuniperrcConfigSP = DocumentDBConfig::JuniperrcConfigSP; + using ImportedFieldsConfigSP = DocumentDBConfig::ImportedFieldsConfigSP; using MaintenanceConfigSP = DocumentDBConfig::MaintenanceConfigSP; DocumentDBConfig::SP current = _pendingConfigSnapshot; @@ -148,6 +151,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) SummaryConfigSP newSummaryConfig; SummarymapConfigSP newSummarymapConfig; JuniperrcConfigSP newJuniperrcConfig; + ImportedFieldsConfigSP newImportedFieldsConfig; MaintenanceConfigSP oldMaintenanceConfig; MaintenanceConfigSP newMaintenanceConfig; @@ -155,18 +159,21 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) if (_bootstrapConfig->getDocumenttypesConfigSP().get() == NULL || _bootstrapConfig->getDocumentTypeRepoSP().get() == NULL || _bootstrapConfig->getProtonConfigSP().get() == NULL || - _bootstrapConfig->getTuneFileDocumentDBSP().get() == NULL) + _bootstrapConfig->getTuneFileDocumentDBSP().get() == NULL) { return; + } } - int64_t generation = snapshot.getGeneration(); LOG(debug, - "Forwarded generation %" PRId64 ", generation %" PRId64, + "Forwarded generation %" + PRId64 + ", generation %" + PRId64, _bootstrapConfig->getGeneration(), generation); - if (!_ignoreForwardedConfig && - _bootstrapConfig->getGeneration() != generation) - return; + if (!_ignoreForwardedConfig && _bootstrapConfig->getGeneration() != generation) { + return; + } int64_t currentGeneration = -1; if (current.get() != NULL) { @@ -177,6 +184,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) newSummaryConfig = current->getSummaryConfigSP(); newSummarymapConfig = current->getSummarymapConfigSP(); newJuniperrcConfig = current->getJuniperrcConfigSP(); + newImportedFieldsConfig = current->getImportedFieldsConfigSP(); oldMaintenanceConfig = current->getMaintenanceConfigSP(); currentGeneration = current->getGeneration(); } @@ -222,25 +230,23 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) "Cannot use bad index schema, validation failed"); abort(); } - newIndexschemaConfig = - IndexschemaConfigSP(indexschemaConfig.release()); + newIndexschemaConfig = IndexschemaConfigSP(indexschemaConfig.release()); + } + if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) { + newAttributesConfig = AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId).release()); + } + if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) { + newSummaryConfig = SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId).release()); + } + if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) { + newSummarymapConfig = SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId).release()); + } + if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) { + newJuniperrcConfig = JuniperrcConfigSP(snapshot.getConfig<JuniperrcConfig>(_configId).release()); + } + if (snapshot.isChanged<ImportedFieldsConfig>(_configId, currentGeneration)) { + newImportedFieldsConfig = ImportedFieldsConfigSP(snapshot.getConfig<ImportedFieldsConfig>(_configId).release()); } - if (snapshot.isChanged<AttributesConfig>(_configId, currentGeneration)) - newAttributesConfig = - AttributesConfigSP(snapshot.getConfig<AttributesConfig>(_configId). - release()); - if (snapshot.isChanged<SummaryConfig>(_configId, currentGeneration)) - newSummaryConfig = - SummaryConfigSP(snapshot.getConfig<SummaryConfig>(_configId). - release()); - if (snapshot.isChanged<SummarymapConfig>(_configId, currentGeneration)) - newSummarymapConfig = - SummarymapConfigSP(snapshot.getConfig<SummarymapConfig>(_configId). - release()); - if (snapshot.isChanged<JuniperrcConfig>(_configId, currentGeneration)) - newJuniperrcConfig = - JuniperrcConfigSP( - snapshot.getConfig<JuniperrcConfig>(_configId).release()); Schema::SP schema(buildSchema(*newAttributesConfig, *newSummaryConfig, @@ -264,6 +270,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) newJuniperrcConfig, _bootstrapConfig->getDocumenttypesConfigSP(), _bootstrapConfig->getDocumentTypeRepoSP(), + newImportedFieldsConfig, _bootstrapConfig->getTuneFileDocumentDBSP(), schema, newMaintenanceConfig, diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 95a1e99d5ab..31c1756f9d2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -369,6 +369,7 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, config::DirSpec spec(snapDir); addEmptyFile(snapDir, "ranking-constants.cfg"); + addEmptyFile(snapDir, "imported-fields.cfg"); DocumentDBConfigHelper dbc(spec, _docTypeName); diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 5deb59aad6f..9dd7a301898 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -8,6 +8,7 @@ #include <vespa/config-indexschema.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/config/config-documenttypes.h> +#include <vespa/config-imported-fields.h> using document::DocumenttypesConfig; using search::TuneFileDocumentDB; @@ -18,6 +19,7 @@ using vespa::config::search::AttributesConfig; using vespa::config::search::SummaryConfig; using vespa::config::search::SummarymapConfig; using vespa::config::search::summary::JuniperrcConfig; +using vespa::config::search::ImportedFieldsConfig; namespace proton { namespace test { @@ -36,6 +38,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, _juniperrc(std::make_shared<JuniperrcConfig>()), _documenttypes(std::make_shared<DocumenttypesConfig>()), _repo(std::make_shared<document::DocumentTypeRepo>()), + _importedFields(std::make_shared<ImportedFieldsConfig>()), _tuneFileDocumentDB(std::make_shared<TuneFileDocumentDB>()), _schema(schema), _maintenance(std::make_shared<DocumentDBMaintenanceConfig>()), @@ -58,6 +61,7 @@ DocumentDBConfigBuilder::build() _juniperrc, _documenttypes, _repo, + _importedFields, _tuneFileDocumentDB, _schema, _maintenance, diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 64f44fc4f63..57f7a3b9948 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -22,6 +22,7 @@ private: DocumentDBConfig::JuniperrcConfigSP _juniperrc; DocumentDBConfig::DocumenttypesConfigSP _documenttypes; document::DocumentTypeRepo::SP _repo; + DocumentDBConfig::ImportedFieldsConfigSP _importedFields; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; search::index::Schema::SP _schema; DocumentDBConfig::MaintenanceConfigSP _maintenance; |