diff options
author | Arne H Juul <arnej@yahoo-inc.com> | 2016-08-26 15:18:27 +0200 |
---|---|---|
committer | Arne H Juul <arnej@yahoo-inc.com> | 2016-08-31 10:37:15 +0200 |
commit | a20eb6eab5fe27128055f0f88166eec05e0413ac (patch) | |
tree | c958c9e3df840b07480ddfad7bf5138f68eb1035 | |
parent | bca73f4f4e56d0a016b99b3917a075823c193c57 (diff) |
subscribe to ranking constants config
* move the .def file for filedistributorrpc to fileacquirerer, and
compile a minimal C++ library from it that can be used by both searchcore
and file distribution
* proton now subscribes to filedistributorrpc as a bootstrap config
* and subscribes to ranking-constants as part of the documentdb config
* when loading a config snapshot, make an empty ranking-constants.cfg
if it did not already exist
32 files changed, 173 insertions, 17 deletions
diff --git a/.gitignore b/.gitignore index bd63798125f..ef6bf2d39b7 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,4 @@ Testing /.ninja_log /build.ninja /rules.ninja +*_test_app diff --git a/CMakeLists.txt b/CMakeLists.txt index 0f152f888f6..cc10f7cb73d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -30,6 +30,7 @@ add_subdirectory(documentapi) add_subdirectory(fastlib) add_subdirectory(fastos) add_subdirectory(fbench) +add_subdirectory(fileacquirer) add_subdirectory(filedistribution) add_subdirectory(fnet) add_subdirectory(frtstream) diff --git a/fileacquirer/CMakeLists.txt b/fileacquirer/CMakeLists.txt new file mode 100644 index 00000000000..54fa8036c66 --- /dev/null +++ b/fileacquirer/CMakeLists.txt @@ -0,0 +1,11 @@ +# Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +vespa_define_module( + DEPENDS + fastos + vespalog + vespalib + config_cloudconfig + + LIBS + src/vespa/fileacquirer +) diff --git a/fileacquirer/pom.xml b/fileacquirer/pom.xml index f6269b29b01..4455950f95a 100644 --- a/fileacquirer/pom.xml +++ b/fileacquirer/pom.xml @@ -27,4 +27,21 @@ <version>${project.version}</version> </dependency> </dependencies> + <build> + <plugins> + <plugin> + <groupId>com.yahoo.vespa</groupId> + <artifactId>config-class-plugin</artifactId> + <version>${project.version}</version> + <executions> + <execution> + <id>config-gen</id> + <goals> + <goal>config-gen</goal> + </goals> + </execution> + </executions> + </plugin> + </plugins> + </build> </project> diff --git a/filedistribution/src/vespa/filedistribution/rpcconfig/filedistributorrpc.def b/fileacquirer/src/main/resources/configdefinitions/filedistributorrpc.def index ceadd8ae8c8..ceadd8ae8c8 100644 --- a/filedistribution/src/vespa/filedistribution/rpcconfig/filedistributorrpc.def +++ b/fileacquirer/src/main/resources/configdefinitions/filedistributorrpc.def diff --git a/fileacquirer/src/vespa/fileacquirer/.gitignore b/fileacquirer/src/vespa/fileacquirer/.gitignore new file mode 100644 index 00000000000..ca425c74c7c --- /dev/null +++ b/fileacquirer/src/vespa/fileacquirer/.gitignore @@ -0,0 +1,2 @@ +config-filedistributorrpc.cpp +config-filedistributorrpc.h diff --git a/filedistribution/src/vespa/filedistribution/rpcconfig/CMakeLists.txt b/fileacquirer/src/vespa/fileacquirer/CMakeLists.txt index e3b90c464c3..6afe392dddd 100644 --- a/filedistribution/src/vespa/filedistribution/rpcconfig/CMakeLists.txt +++ b/fileacquirer/src/vespa/fileacquirer/CMakeLists.txt @@ -1,7 +1,7 @@ # Copyright 2016 Yahoo Inc. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. -vespa_add_library(filedistribution_filedistributorrpcconfig STATIC +vespa_add_library(fileacquirer STATIC SOURCES DEPENDS ) -vespa_generate_config(filedistribution_filedistributorrpcconfig filedistributorrpc.def) +vespa_generate_config(fileacquirer filedistributorrpc.def) install(FILES filedistributorrpc.def DESTINATION var/db/vespa/config_server/serverdb/classes) diff --git a/fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def b/fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def new file mode 120000 index 00000000000..c12c0b8299e --- /dev/null +++ b/fileacquirer/src/vespa/fileacquirer/filedistributorrpc.def @@ -0,0 +1 @@ +../../main/resources/configdefinitions/filedistributorrpc.def
\ No newline at end of file diff --git a/filedistribution/CMakeLists.txt b/filedistribution/CMakeLists.txt index 6205f44515b..b5b16823f08 100644 --- a/filedistribution/CMakeLists.txt +++ b/filedistribution/CMakeLists.txt @@ -10,6 +10,7 @@ vespa_define_module( staging_vespalib vespadefaults vespalib + fileacquirer EXTERNAL_DEPENDS torrent-rasterbar @@ -18,7 +19,6 @@ vespa_define_module( src/vespa/filedistribution/rpc src/vespa/filedistribution/common src/vespa/filedistribution/manager - src/vespa/filedistribution/rpcconfig src/vespa/filedistribution/distributor src/vespa/filedistribution/model diff --git a/filedistribution/pom.xml b/filedistribution/pom.xml index d6885bc0667..c186f4928bb 100644 --- a/filedistribution/pom.xml +++ b/filedistribution/pom.xml @@ -29,8 +29,7 @@ <configuration> <defFilesDirectories> src/vespa/filedistribution/distributor, - src/vespa/filedistribution/model, - src/vespa/filedistribution/rpcconfig + src/vespa/filedistribution/model </defFilesDirectories> </configuration> <executions> diff --git a/filedistribution/src/apps/filedistributor/CMakeLists.txt b/filedistribution/src/apps/filedistributor/CMakeLists.txt index f7be9d46c76..ce63a47b143 100644 --- a/filedistribution/src/apps/filedistributor/CMakeLists.txt +++ b/filedistribution/src/apps/filedistributor/CMakeLists.txt @@ -7,7 +7,6 @@ vespa_add_executable(filedistribution_filedistributor_app DEPENDS filedistribution_distributor filedistribution_filedistributionmodel - filedistribution_filedistributorrpcconfig filedistribution_filedistributorrpc filedistribution_common ) diff --git a/filedistribution/src/apps/filedistributor/filedistributor.cpp b/filedistribution/src/apps/filedistributor/filedistributor.cpp index 6a7e5e58cbf..4457cf28ee3 100644 --- a/filedistribution/src/apps/filedistributor/filedistributor.cpp +++ b/filedistribution/src/apps/filedistributor/filedistributor.cpp @@ -17,7 +17,7 @@ #include <vespa/fastos/app.h> #include <vespa/config-zookeepers.h> -#include <vespa/filedistribution/rpcconfig/config-filedistributorrpc.h> +#include <vespa/fileacquirer/config-filedistributorrpc.h> #include <vespa/filedistribution/distributor/config-filedistributor.h> #include <vespa/filedistribution/model/config-filereferences.h> diff --git a/searchcore/CMakeLists.txt b/searchcore/CMakeLists.txt index 2aabbeeabfd..aa1b2c1f4c4 100644 --- a/searchcore/CMakeLists.txt +++ b/searchcore/CMakeLists.txt @@ -18,6 +18,7 @@ vespa_define_module( searchcorespi searchsummary storageserver_storageapp + fileacquirer LIBS src/vespa/searchcore/config diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 576b283572b..7f7dde5d402 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -25,6 +25,7 @@ using namespace proton; using namespace vespa::config::search; using namespace vespa::config::search::core; using namespace vespa::config::search::summary; +using namespace cloud::config::filedistribution; using std::shared_ptr; using document::DocumentType; @@ -100,6 +101,7 @@ public: } typedef DocumentDBConfig CS; CS::RankProfilesConfigSP rankProfiles(new RankProfilesConfig()); + CS::RankingConstantsConfigSP rankingConstants(new RankingConstantsConfig()); CS::IndexschemaConfigSP indexschema = _schemaFactory->createIndexSchema(*docType); CS::AttributesConfigSP attributes = _schemaFactory->createAttributes(*docType); CS::SummaryConfigSP summary = _schemaFactory->createSummary(*docType); @@ -114,6 +116,7 @@ public: return DocumentDBConfig::SP(new DocumentDBConfig( 1, rankProfiles, + rankingConstants, indexschema, attributes, summary, @@ -170,6 +173,7 @@ public: factory.getTypeCfg(), factory.getTypeRepo(), BootstrapConfig::ProtonConfigSP(new ProtonConfig()), + BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()), tuneFileDocDB)); mgr.forwardConfig(b); mgr.nextGeneration(0); diff --git a/searchcore/src/tests/proton/config/config.cpp b/searchcore/src/tests/proton/config/config.cpp index 3cb6f3c3e23..5a26e3ac016 100644 --- a/searchcore/src/tests/proton/config/config.cpp +++ b/searchcore/src/tests/proton/config/config.cpp @@ -10,12 +10,14 @@ LOG_SETUP("config_test"); #include <vespa/searchcore/proton/server/documentdbconfigmanager.h> #include <vespa/searchcore/proton/server/protonconfigurer.h> #include <vespa/vespalib/util/varholder.h> +#include <vespa/fileacquirer/config-filedistributorrpc.h> using namespace config; using namespace proton; using namespace vespa::config::search::core; using namespace vespa::config::search::summary; using namespace vespa::config::search; +using namespace cloud::config::filedistribution; using config::ConfigUri; using document::DocumentTypeRepo; @@ -30,6 +32,7 @@ struct DoctypeFixture { typedef vespalib::LinkedPtr<DoctypeFixture> LP; AttributesConfigBuilder attributesBuilder; RankProfilesConfigBuilder rankProfilesBuilder; + RankingConstantsConfigBuilder rankingConstantsBuilder; IndexschemaConfigBuilder indexschemaBuilder; SummaryConfigBuilder summaryBuilder; SummarymapConfigBuilder summarymapBuilder; @@ -40,6 +43,7 @@ struct ConfigTestFixture { const std::string configId; ProtonConfigBuilder protonBuilder; DocumenttypesConfigBuilder documenttypesBuilder; + FiledistributorrpcConfigBuilder filedistBuilder; map<std::string, DoctypeFixture::LP> dbConfig; ConfigSet set; IConfigContext::SP context; @@ -49,6 +53,7 @@ struct ConfigTestFixture { : configId(id), protonBuilder(), documenttypesBuilder(), + filedistBuilder(), dbConfig(), set(), context(new ConfigContext(set)), @@ -56,6 +61,7 @@ struct ConfigTestFixture { { set.addBuilder(configId, &protonBuilder); set.addBuilder(configId, &documenttypesBuilder); + set.addBuilder(configId, &filedistBuilder); addDocType("_alwaysthere_"); } @@ -77,6 +83,7 @@ struct ConfigTestFixture { DoctypeFixture::LP fixture(new DoctypeFixture()); set.addBuilder(db.configid, &fixture->attributesBuilder); set.addBuilder(db.configid, &fixture->rankProfilesBuilder); + set.addBuilder(db.configid, &fixture->rankingConstantsBuilder); set.addBuilder(db.configid, &fixture->indexschemaBuilder); set.addBuilder(db.configid, &fixture->summaryBuilder); set.addBuilder(db.configid, &fixture->summarymapBuilder); @@ -111,6 +118,7 @@ struct ConfigTestFixture { DoctypeFixture::LP fixture(dbConfig[name]); return (fixture->attributesBuilder == dbc->getAttributesConfig() && fixture->rankProfilesBuilder == dbc->getRankProfilesConfig() && + fixture->rankingConstantsBuilder == dbc->getRankingConstantsConfig() && fixture->indexschemaBuilder == dbc->getIndexschemaConfig() && fixture->summaryBuilder == dbc->getSummaryConfig() && fixture->summarymapBuilder == dbc->getSummarymapConfig() && @@ -127,6 +135,7 @@ struct ConfigTestFixture { BootstrapConfig::DocumenttypesConfigSP(new DocumenttypesConfig(documenttypesBuilder)), DocumentTypeRepo::SP(new DocumentTypeRepo(documenttypesBuilder)), BootstrapConfig::ProtonConfigSP(new ProtonConfig(protonBuilder)), + BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()), TuneFileDocumentDB::SP(new TuneFileDocumentDB()))); } @@ -166,7 +175,7 @@ typedef OwnerFixture<DocumentDBConfig::SP, IDocumentDBConfigOwner> DBOwner; TEST_F("require that bootstrap config manager creats correct key set", BootstrapConfigManager("foo")) { const ConfigKeySet set(f1.createConfigKeySet()); - ASSERT_EQUAL(2u, set.size()); + ASSERT_EQUAL(3u, set.size()); ConfigKey protonKey(ConfigKey::create<ProtonConfig>("foo")); ConfigKey dtKey(ConfigKey::create<DocumenttypesConfig>("foo")); ASSERT_TRUE(set.find(protonKey) != set.end()); @@ -196,7 +205,7 @@ TEST_FF("require_that_documentdb_config_manager_subscribes_for_config", DocumentDBConfigManager(f1.configId + "/typea", "typea")) { f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); - ASSERT_EQUAL(6u, keySet.size()); + ASSERT_EQUAL(7u, keySet.size()); ConfigRetriever retriever(keySet, f1.context); f2.forwardConfig(f1.getBootstrapConfig(1)); f2.update(retriever.getBootstrapConfigs()); // Cheating, but we only need the configs diff --git a/searchcore/src/tests/proton/docsummary/docsummary.cpp b/searchcore/src/tests/proton/docsummary/docsummary.cpp index b472c612281..e5365ba0ede 100644 --- a/searchcore/src/tests/proton/docsummary/docsummary.cpp +++ b/searchcore/src/tests/proton/docsummary/docsummary.cpp @@ -31,6 +31,7 @@ using namespace search::docsummary; using namespace search::engine; using namespace search::index; using namespace search::transactionlog; +using namespace cloud::config::filedistribution; using search::TuneFileDocumentDB; using document::DocumenttypesConfig; using storage::spi::Timestamp; @@ -169,6 +170,7 @@ public: _documenttypesConfig, _repo, BootstrapConfig::ProtonConfigSP(new ProtonConfig()), + BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()), _tuneFileDocumentDB)); _configMgr.forwardConfig(b); _configMgr.nextGeneration(0); diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 4017364eb41..e401f8e0301 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -320,8 +320,8 @@ createConfig() DocumentDBConfig::SP config (new DocumentDBConfig( 0, - DocumentDBConfig::RankProfilesConfigSP( - new RankProfilesConfig()), + DocumentDBConfig::RankProfilesConfigSP(new RankProfilesConfig()), + DocumentDBConfig::RankingConstantsConfigSP(new RankingConstantsConfig()), DocumentDBConfig::IndexschemaConfigSP(new IndexschemaConfig()), DocumentDBConfig::AttributesConfigSP(new AttributesConfig()), DocumentDBConfig::SummaryConfigSP(new SummaryConfig()), @@ -344,6 +344,7 @@ createConfig(const Schema::SP &schema) (new DocumentDBConfig( 0, DocumentDBConfig::RankProfilesConfigSP(new RankProfilesConfig()), + DocumentDBConfig::RankingConstantsConfigSP(new RankingConstantsConfig()), DocumentDBConfig::IndexschemaConfigSP(new IndexschemaConfig()), DocumentDBConfig::AttributesConfigSP(new AttributesConfig()), DocumentDBConfig::SummaryConfigSP(new SummaryConfig()), diff --git a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp index 5c6a7db1876..a98f07439c9 100644 --- a/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp +++ b/searchcore/src/tests/proton/documentdb/document_subdbs/document_subdbs_test.cpp @@ -31,6 +31,7 @@ using namespace search; using namespace search::common; using namespace search::index; using namespace search::transactionlog; +using namespace cloud::config::filedistribution; using namespace searchcorespi; using namespace vespalib; using proton::bucketdb::BucketDBHandler; @@ -262,6 +263,7 @@ struct MyConfigSnapshot documenttypesConfig, _builder.getDocumentTypeRepo(), BootstrapConfig::ProtonConfigSP(new ProtonConfig()), + BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()), tuneFileDocumentDB)); config::DirSpec spec(cfgDir); DocumentDBConfigHelper mgr(spec, "searchdocument"); diff --git a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp index 605a8fc5530..d6d54d31f16 100644 --- a/searchcore/src/tests/proton/documentdb/documentdb_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdb_test.cpp @@ -28,6 +28,7 @@ using search::index::Schema; using search::transactionlog::TransLogServer; using namespace proton; using namespace vespalib::slime; +using namespace cloud::config::filedistribution; using search::TuneFileDocumentDB; using document::DocumenttypesConfig; using search::index::DummyFileHeaderContext; @@ -79,6 +80,7 @@ Fixture::Fixture() documenttypesConfig, repo, BootstrapConfig::ProtonConfigSP(new ProtonConfig()), + BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()), tuneFileDocumentDB)); mgr.forwardConfig(b); mgr.nextGeneration(0); diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index 5f9de9ae545..831c5d522d9 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -14,6 +14,7 @@ using namespace search::index; using namespace vespa::config::search; using std::shared_ptr; using std::make_shared; +using vespa::config::search::core::RankingConstantsConfig; typedef shared_ptr<DocumentDBConfig> DDBCSP; @@ -28,6 +29,7 @@ getConfig(int64_t generation, const Schema::SP &schema, return make_shared<DocumentDBConfig>( generation, make_shared<RankProfilesConfig>(rankProfiles), + make_shared<RankingConstantsConfig>(), make_shared<IndexschemaConfig>(), make_shared<AttributesConfig>(), make_shared<SummaryConfig>(), diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp index e935fff1431..bb0a3e33b50 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfigscout/documentdbconfigscout_test.cpp @@ -15,6 +15,7 @@ using namespace search::index; using namespace vespa::config::search; using std::shared_ptr; using std::make_shared; +using vespa::config::search::core::RankingConstantsConfig; typedef shared_ptr<DocumentDBConfig> DDBCSP; @@ -29,6 +30,7 @@ getConfig(int64_t generation, const Schema::SP &schema, return make_shared<DocumentDBConfig>( generation, make_shared<RankProfilesConfig>(), + make_shared<RankingConstantsConfig>(), make_shared<IndexschemaConfig>(), make_shared<AttributesConfig>(attributes), make_shared<SummaryConfig>(), diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 499f8a11aaf..c811c8912b0 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -18,6 +18,7 @@ using namespace search::index; using namespace search; using namespace vespa::config::search::core; using namespace vespa::config::search; +using namespace cloud::config::filedistribution; typedef DocumentDBConfigHelper DBCM; typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP; @@ -32,6 +33,7 @@ DocumentDBConfig::SP getConfig(int64_t generation, const Schema::SP &schema) { typedef DocumentDBConfig::RankProfilesConfigSP RankProfilesConfigSP; + typedef DocumentDBConfig::RankingConstantsConfigSP RankingConstantsConfigSP; typedef DocumentDBConfig::IndexschemaConfigSP IndexschemaConfigSP; typedef DocumentDBConfig::AttributesConfigSP AttributesConfigSP; typedef DocumentDBConfig::SummaryConfigSP SummaryConfigSP; @@ -40,6 +42,7 @@ getConfig(int64_t generation, const Schema::SP &schema) typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP; RankProfilesConfigSP rp(new vespa::config::search::RankProfilesConfig); + RankingConstantsConfigSP rc(new vespa::config::search::core::RankingConstantsConfig); IndexschemaConfigSP is(new vespa::config::search::IndexschemaConfig); AttributesConfigSP a(new vespa::config::search::AttributesConfig); SummaryConfigSP s(new vespa::config::search::SummaryConfig); @@ -54,6 +57,7 @@ getConfig(int64_t generation, const Schema::SP &schema) new DocumentDBConfig( generation, rp, + rc, is, a, s, @@ -95,6 +99,7 @@ makeBaseConfigSnapshot() dtcfg, DocumentTypeRepo::SP(new DocumentTypeRepo(*dtcfg)), BootstrapConfig::ProtonConfigSP(new ProtonConfig()), + BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()), TuneFileDocumentDB::SP(new TuneFileDocumentDB()))); dbcm.forwardConfig(b); dbcm.nextGeneration(0); @@ -126,6 +131,7 @@ makeEmptyConfigSnapshot(void) return DocumentDBConfig::SP(new DocumentDBConfig( 0, DocumentDBConfig::RankProfilesConfigSP(), + DocumentDBConfig::RankingConstantsConfigSP(), DocumentDBConfig::IndexschemaConfigSP(), DocumentDBConfig::AttributesConfigSP(), DocumentDBConfig::SummaryConfigSP(), @@ -159,6 +165,7 @@ void assertEqualSnapshot(const DocumentDBConfig &exp, const DocumentDBConfig &act) { EXPECT_TRUE(exp.getRankProfilesConfig() == act.getRankProfilesConfig()); + EXPECT_TRUE(exp.getRankingConstantsConfig() == act.getRankingConstantsConfig()); EXPECT_TRUE(exp.getIndexschemaConfig() == act.getIndexschemaConfig()); EXPECT_TRUE(exp.getAttributesConfig() == act.getAttributesConfig()); EXPECT_TRUE(exp.getSummaryConfig() == act.getSummaryConfig()); diff --git a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp index 301633404bc..f1cbd012ab5 100644 --- a/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp +++ b/searchcore/src/tests/proton/server/memoryconfigstore_test.cpp @@ -24,6 +24,7 @@ getConfig(int64_t generation, const Schema::SP &schema) new DocumentDBConfig( generation, DocumentDBConfig::RankProfilesConfigSP(), + DocumentDBConfig::RankingConstantsConfigSP(), DocumentDBConfig::IndexschemaConfigSP(), DocumentDBConfig::AttributesConfigSP(), DocumentDBConfig::SummaryConfigSP(), diff --git a/searchcore/src/vespa/searchcore/config/proton.def b/searchcore/src/vespa/searchcore/config/proton.def index 3335af2f02d..bd89c7ec460 100644 --- a/searchcore/src/vespa/searchcore/config/proton.def +++ b/searchcore/src/vespa/searchcore/config/proton.def @@ -272,9 +272,12 @@ tlsconfigid string default="" restart ## Slobrok configid slobrokconfigid string default="" restart -## Slobrok configid +## Routing configid routingconfigid string default="" restart +## filedistributor rpc configuration +filedistributor.configid reference default="" restart + ## Interval between pruning of old removed documents. ## ## Default value is 6 hours (21600 seconds). diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp index f4b5757f558..f019199348d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.cpp @@ -27,10 +27,12 @@ BootstrapConfig::BootstrapConfig( const DocumenttypesConfigSP &documenttypes, const DocumentTypeRepo::SP &repo, const ProtonConfigSP &protonConfig, + const FiledistributorrpcConfigSP &filedistRpcConfSP, const search::TuneFileDocumentDB::SP &tuneFileDocumentDB) : _documenttypes(documenttypes), _repo(repo), _proton(protonConfig), + _fileDistributorRpc(filedistRpcConfSP), _tuneFileDocumentDB(tuneFileDocumentDB), _generation(generation) { diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h index a222c51920a..c104d57e4ae 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfig.h @@ -9,6 +9,7 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> +#include <vespa/fileacquirer/config-filedistributorrpc.h> namespace proton { @@ -20,12 +21,15 @@ class BootstrapConfig public: typedef std::shared_ptr<BootstrapConfig> SP; typedef std::shared_ptr<vespa::config::search::core::ProtonConfig> ProtonConfigSP; + typedef std::shared_ptr<cloud::config::filedistribution::FiledistributorrpcConfig> FiledistributorrpcConfigSP; + typedef DocumentDBConfig::DocumenttypesConfigSP DocumenttypesConfigSP; private: DocumenttypesConfigSP _documenttypes; document::DocumentTypeRepo::SP _repo; ProtonConfigSP _proton; + FiledistributorrpcConfigSP _fileDistributorRpc; search::TuneFileDocumentDB::SP _tuneFileDocumentDB; int64_t _generation; @@ -34,6 +38,7 @@ public: const DocumenttypesConfigSP & documenttypes, const document::DocumentTypeRepo::SP &repo, const ProtonConfigSP &protonConfig, + const FiledistributorrpcConfigSP &filedistRpcConfSP, const search::TuneFileDocumentDB::SP & _tuneFileDocumentDB); @@ -43,6 +48,12 @@ public: return *_documenttypes; } + const cloud::config::filedistribution::FiledistributorrpcConfig & + getFiledistributorrpcConfig() const { return *_fileDistributorRpc; } + + const FiledistributorrpcConfigSP & + getFiledistributorrpcConfigSP() const { return _fileDistributorRpc; } + const DocumenttypesConfigSP & getDocumenttypesConfigSP(void) const { diff --git a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp index 555e2f2797f..7e9da69d09f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bootstrapconfigmanager.cpp @@ -4,12 +4,14 @@ #include "bootstrapconfigmanager.h" #include <vespa/log/log.h> LOG_SETUP(".proton.server.bootstrapconfigmanager"); +#include <vespa/fileacquirer/config-filedistributorrpc.h> using namespace vespa::config::search; using namespace config; using document::DocumentTypeRepo; using search::TuneFileDocumentDB; using vespa::config::search::core::ProtonConfig; +using cloud::config::filedistribution::FiledistributorrpcConfig; using document::DocumenttypesConfig; using config::SourceSpec; @@ -28,7 +30,9 @@ BootstrapConfigManager(const vespalib::string & configId) const ConfigKeySet BootstrapConfigManager::createConfigKeySet() const { - return ConfigKeySet().add<ProtonConfig, DocumenttypesConfig>(_configId); + return ConfigKeySet().add<ProtonConfig>(_configId) + .add<DocumenttypesConfig>(_configId) + .add<FiledistributorrpcConfig>(_configId); } void @@ -38,6 +42,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) typedef BootstrapConfig::DocumenttypesConfigSP DocumenttypesConfigSP; ProtonConfigSP newProtonConfig; + BootstrapConfig::FiledistributorrpcConfigSP newFiledistRpcConfSP; TuneFileDocumentDB::SP newTuneFileDocumentDB; DocumenttypesConfigSP newDocumenttypesConfig; DocumentTypeRepo::SP newRepo; @@ -46,6 +51,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) BootstrapConfig::SP current = _pendingConfigSnapshot; if (current.get() != NULL) { newProtonConfig = current->getProtonConfigSP(); + newFiledistRpcConfSP = current->getFiledistributorrpcConfigSP(); newTuneFileDocumentDB = current->getTuneFileDocumentDBSP(); newDocumenttypesConfig = current->getDocumenttypesConfigSP(); newRepo = current->getDocumentTypeRepoSP(); @@ -80,6 +86,12 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) newTuneFileDocumentDB = tuneFileDocumentDB; } + if (snapshot.isChanged<FiledistributorrpcConfig>(_configId, currentGen)) { + LOG(info, "Filedistributorrpc config is changed"); + auto p = snapshot.getConfig<FiledistributorrpcConfig>(_configId); + newFiledistRpcConfSP = BootstrapConfig::FiledistributorrpcConfigSP(p.release()); + } + if (snapshot.isChanged<DocumenttypesConfig>(_configId, currentGen)) { LOG(spam, "Documenttypes config is changed"); std::unique_ptr<DocumenttypesConfig> documenttypesConfig = @@ -90,6 +102,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) newRepo = repo; } assert(newProtonConfig.get() != NULL); + assert(newFiledistRpcConfSP.get() != NULL); assert(newTuneFileDocumentDB.get() != NULL); assert(newDocumenttypesConfig.get() != NULL); assert(newRepo.get() != NULL); @@ -99,6 +112,7 @@ BootstrapConfigManager::update(const ConfigSnapshot & snapshot) newDocumenttypesConfig, newRepo, newProtonConfig, + newFiledistRpcConfSP, newTuneFileDocumentDB)); assert(newSnapshot->valid()); diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index a2487671134..91e30ebebe2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -12,11 +12,13 @@ using document::DocumenttypesConfig; using search::TuneFileDocumentDB; using search::index::Schema; using vespa::config::search::SummarymapConfig; +using vespa::config::search::core::RankingConstantsConfig; namespace proton { DocumentDBConfig::ComparisonResult::ComparisonResult() : rankProfilesChanged(false), + rankingConstantsChanged(false), indexschemaChanged(false), attributesChanged(false), summaryChanged(false), @@ -33,6 +35,7 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() DocumentDBConfig::DocumentDBConfig( int64_t generation, const RankProfilesConfigSP &rankProfiles, + const RankingConstantsConfigSP &rankingConstants, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, const SummaryConfigSP &summary, @@ -50,6 +53,7 @@ DocumentDBConfig::DocumentDBConfig( _docTypeName(docTypeName), _generation(generation), _rankProfiles(rankProfiles), + _rankingConstants(rankingConstants), _indexschema(indexschema), _attributes(attributes), _summary(summary), @@ -72,6 +76,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _docTypeName(cfg._docTypeName), _generation(cfg._generation), _rankProfiles(cfg._rankProfiles), + _rankingConstants(cfg._rankingConstants), _indexschema(cfg._indexschema), _attributes(cfg._attributes), _summary(cfg._summary), @@ -92,6 +97,8 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const { return equals<RankProfilesConfig>(_rankProfiles.get(), rhs._rankProfiles.get()) && + equals<RankingConstantsConfig>(_rankingConstants.get(), + rhs._rankingConstants.get()) && equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get()) && equals<AttributesConfig>(_attributes.get(), @@ -119,8 +126,9 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const { ComparisonResult retval; retval.rankProfilesChanged = - !equals<RankProfilesConfig>(_rankProfiles.get(), - rhs._rankProfiles.get()); + !equals<RankProfilesConfig>(_rankProfiles.get(), rhs._rankProfiles.get()); + retval.rankingConstantsChanged = + !equals<RankingConstantsConfig>(_rankingConstants.get(), rhs._rankingConstants.get()); retval.indexschemaChanged = !equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get()); retval.attributesChanged = @@ -151,6 +159,7 @@ bool DocumentDBConfig::valid(void) const { return _rankProfiles.get() != NULL && + _rankingConstants.get() != NULL && _indexschema.get() != NULL && _attributes.get() != NULL && _summary.get() != NULL && @@ -189,6 +198,7 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) SP ret = std::make_shared<DocumentDBConfig>( o._generation, emptyConfig(o._rankProfiles), + emptyConfig(o._rankingConstants), o._indexschema, o._attributes, o._summary, @@ -227,6 +237,7 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) return std::make_shared<DocumentDBConfig>( _generation, _rankProfiles, + _rankingConstants, _indexschema, attributes, _summary, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 659b11ef8c1..5c0e12ae964 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -15,6 +15,7 @@ #include <vespa/config-summarymap.h> #include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> +#include <vespa/searchcore/config/config-ranking-constants.h> namespace proton { @@ -25,6 +26,7 @@ public: { public: bool rankProfilesChanged; + bool rankingConstantsChanged; bool indexschemaChanged; bool attributesChanged; bool summaryChanged; @@ -43,6 +45,7 @@ public: typedef std::shared_ptr<vespa::config::search::IndexschemaConfig> IndexschemaConfigSP; typedef std::shared_ptr<vespa::config::search::AttributesConfig> AttributesConfigSP; typedef std::shared_ptr<vespa::config::search::RankProfilesConfig> RankProfilesConfigSP; + typedef std::shared_ptr<vespa::config::search::core::RankingConstantsConfig> RankingConstantsConfigSP; typedef std::shared_ptr<vespa::config::search::SummaryConfig> SummaryConfigSP; typedef std::shared_ptr<vespa::config::search::SummarymapConfig> SummarymapConfigSP; typedef std::shared_ptr<vespa::config::search::summary::JuniperrcConfig> JuniperrcConfigSP; @@ -54,6 +57,7 @@ private: vespalib::string _docTypeName; int64_t _generation; RankProfilesConfigSP _rankProfiles; + RankingConstantsConfigSP _rankingConstants; IndexschemaConfigSP _indexschema; AttributesConfigSP _attributes; SummaryConfigSP _summary; @@ -78,6 +82,7 @@ private: public: DocumentDBConfig(int64_t generation, const RankProfilesConfigSP &rankProfiles, + const RankingConstantsConfigSP &rankingConstants, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, const SummaryConfigSP &summary, @@ -104,6 +109,9 @@ public: const vespa::config::search::RankProfilesConfig & getRankProfilesConfig() const { return *_rankProfiles; } + const vespa::config::search::core::RankingConstantsConfig & + getRankingConstantsConfig() const { return *_rankingConstants; } + const vespa::config::search::IndexschemaConfig & getIndexschemaConfig() const { return *_indexschema; } @@ -125,6 +133,9 @@ public: const RankProfilesConfigSP & getRankProfilesConfigSP(void) const { return _rankProfiles; } + const RankingConstantsConfigSP & + getRankingConstantsConfigSP() const { return _rankingConstants; } + const IndexschemaConfigSP & getIndexschemaConfigSP(void) const { return _indexschema; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index d566949eb3b..9ffd9889b73 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -19,6 +19,10 @@ using search::index::Schema; using search::index::SchemaBuilder; using fastos::TimeStamp; +// RankingConstantsConfigBuilder +// RankingConstantsConfig + + namespace proton { const ConfigKeySet @@ -26,6 +30,7 @@ DocumentDBConfigManager::createConfigKeySet() const { ConfigKeySet set; set.add<RankProfilesConfig, + RankingConstantsConfig, IndexschemaConfig, AttributesConfig, SummaryConfig, @@ -123,6 +128,7 @@ void DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) { typedef DocumentDBConfig::RankProfilesConfigSP RankProfilesConfigSP; + typedef DocumentDBConfig::RankingConstantsConfigSP RankingConstantsConfigSP; typedef DocumentDBConfig::IndexschemaConfigSP IndexschemaConfigSP; typedef DocumentDBConfig::AttributesConfigSP AttributesConfigSP; typedef DocumentDBConfig::SummaryConfigSP SummaryConfigSP; @@ -132,6 +138,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) DocumentDBConfig::SP current = _pendingConfigSnapshot; RankProfilesConfigSP newRankProfilesConfig; + RankingConstantsConfigSP newRankingConstantsConfig; IndexschemaConfigSP newIndexschemaConfig; AttributesConfigSP newAttributesConfig; SummaryConfigSP newSummaryConfig; @@ -160,6 +167,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) int64_t currentGeneration = -1; if (current.get() != NULL) { newRankProfilesConfig = current->getRankProfilesConfigSP(); + newRankingConstantsConfig = current->getRankingConstantsConfigSP(); newIndexschemaConfig = current->getIndexschemaConfigSP(); newAttributesConfig = current->getAttributesConfigSP(); newSummaryConfig = current->getSummaryConfigSP(); @@ -169,11 +177,18 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) currentGeneration = current->getGeneration(); } - if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) + if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { newRankProfilesConfig = RankProfilesConfigSP( snapshot.getConfig<RankProfilesConfig>(_configId). release()); + } + if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { + newRankingConstantsConfig = + RankingConstantsConfigSP( + snapshot.getConfig<RankingConstantsConfig>(_configId) + .release()); + } if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) { std::unique_ptr<IndexschemaConfig> indexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); @@ -218,6 +233,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot & snapshot) DocumentDBConfig::SP newSnapshot( new DocumentDBConfig(generation, newRankProfilesConfig, + newRankingConstantsConfig, newIndexschemaConfig, newAttributesConfig, newSummaryConfig, diff --git a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp index 8b5a84d3f9e..1a21127a87d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/fileconfigmanager.cpp @@ -20,6 +20,7 @@ using document::DocumenttypesConfig; using search::IndexMetaInfo; using search::SerialNum; using search::index::Schema; +using cloud::config::filedistribution::FiledistributorrpcConfig; using vespa::config::search::AttributesConfig; using vespa::config::search::IndexschemaConfig; using vespa::config::search::RankProfilesConfig; @@ -336,6 +337,25 @@ FileConfigManager::saveConfig(const DocumentDBConfig &snapshot, (void) saveValidSnap; } +namespace { + +// add an empty file if it's not already present +void addEmptyFile(vespalib::string snapDir, vespalib::string fileName) +{ + vespalib::string path = snapDir + "/" + fileName; + if (access(path.c_str(), R_OK) == 0) { + // exists OK + return; + } + int fd = creat(path.c_str(), 0444); + if (fd < 0) { + LOG(error, "Could not create empty file '%s': %s", path.c_str(), strerror(errno)); + return; + } + close(fd); +} + +} void FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, @@ -347,6 +367,8 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, vespalib::string snapDir(_baseDir + "/" + snapDirBaseName); config::DirSpec spec(snapDir); + addEmptyFile(snapDir, "ranking-constants.cfg"); + DocumentDBConfigHelper dbc(spec, _docTypeName); typedef DocumenttypesConfig DTC; @@ -362,6 +384,8 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, repo.reset(new DocumentTypeRepo(*docTypesCfg)); } + auto filedistRpcConf = BootstrapConfig::FiledistributorrpcConfigSP(new FiledistributorrpcConfig()); + /* * XXX: If non-default maintenance config is used then an extra config * snapshot is saved after replaying transaction log due to the use @@ -373,6 +397,7 @@ FileConfigManager::loadConfig(const DocumentDBConfig ¤tSnapshot, docTypesCfg, repo, _protonConfig, + filedistRpcConf, currentSnapshot.getTuneFileDocumentDBSP())); dbc.forwardConfig(bootstrap); dbc.nextGeneration(0); diff --git a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp index db64b57698e..fd89a34250d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/storeonlydocsubdb.cpp @@ -29,7 +29,6 @@ LOG_SETUP(".proton.server.storeonlydocsubdb"); #include <vespa/vespalib/util/closuretask.h> using vespa::config::search::AttributesConfig; -using vespa::config::search::RankProfilesConfig; using vespa::config::search::core::ProtonConfig; using search::GrowStrategy; using search::AttributeGuard; |