diff options
author | Tor Egge <Tor.Egge@online.no> | 2023-05-09 16:31:10 +0200 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2023-05-09 16:31:10 +0200 |
commit | 59ebc5e05ac53e0dee8661741ab0c88b3e751009 (patch) | |
tree | 6d72c7e51053820fef4d86c2faa72f777e2a86a5 /searchcore | |
parent | d26ca8bf4c154f42e57eb41e66c8f057e1f4c26f (diff) |
Add RankingAssetsBuilder.
Diffstat (limited to 'searchcore')
5 files changed, 52 insertions, 104 deletions
diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index d8dafd1c39e..88ccec80b8a 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -58,17 +58,17 @@ struct DBConfigFixture { return DocumentDBConfig::build_schema(_attributesBuilder, _indexschemaBuilder); } - static RankingConstants::SP buildRankingConstants() + static std::shared_ptr<const RankingConstants> buildRankingConstants() { return std::make_shared<RankingConstants>(); } - static RankingExpressions::SP buildRankingExpressions() + static std::shared_ptr<const RankingExpressions> buildRankingExpressions() { return std::make_shared<RankingExpressions>(); } - static OnnxModels::SP buildOnnxModels() + static std::shared_ptr<const OnnxModels> buildOnnxModels() { return std::make_shared<OnnxModels>(); } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index 00d9c194b9e..cc8bc2c8b96 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -55,9 +55,9 @@ DocumentDBConfig::ComparisonResult::ComparisonResult() DocumentDBConfig::DocumentDBConfig( int64_t generation, const RankProfilesConfigSP &rankProfiles, - const RankingConstants::SP &rankingConstants, - const RankingExpressions::SP &rankingExpressions, - const OnnxModels::SP &onnxModels, + const std::shared_ptr<const RankingConstants> &rankingConstants, + const std::shared_ptr<const RankingExpressions> &rankingExpressions, + const std::shared_ptr<const OnnxModels> &onnxModels, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, const SummaryConfigSP &summary, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 11f23cf0b66..848e68ce0f2 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -111,28 +111,28 @@ public: using ImportedFieldsConfigSP = std::shared_ptr<ImportedFieldsConfig>; private: - vespalib::string _configId; - vespalib::string _docTypeName; - int64_t _generation; - RankProfilesConfigSP _rankProfiles; - RankingConstants::SP _rankingConstants; - RankingExpressions::SP _rankingExpressions; - OnnxModels::SP _onnxModels; - IndexschemaConfigSP _indexschema; - AttributesConfigSP _attributes; - SummaryConfigSP _summary; - JuniperrcConfigSP _juniperrc; - DocumenttypesConfigSP _documenttypes; + vespalib::string _configId; + vespalib::string _docTypeName; + int64_t _generation; + RankProfilesConfigSP _rankProfiles; + std::shared_ptr<const RankingConstants> _rankingConstants; + std::shared_ptr<const RankingExpressions> _rankingExpressions; + std::shared_ptr<const OnnxModels> _onnxModels; + IndexschemaConfigSP _indexschema; + AttributesConfigSP _attributes; + SummaryConfigSP _summary; + JuniperrcConfigSP _juniperrc; + DocumenttypesConfigSP _documenttypes; std::shared_ptr<const document::DocumentTypeRepo> _repo; - ImportedFieldsConfigSP _importedFields; - search::TuneFileDocumentDB::SP _tuneFileDocumentDB; - search::index::Schema::SP _schema; - MaintenanceConfigSP _maintenance; - search::LogDocumentStore::Config _storeConfig; - const ThreadingServiceConfig _threading_service_config; - const AllocConfig _alloc_config; - SP _orig; - bool _delayedAttributeAspects; + ImportedFieldsConfigSP _importedFields; + search::TuneFileDocumentDB::SP _tuneFileDocumentDB; + search::index::Schema::SP _schema; + MaintenanceConfigSP _maintenance; + search::LogDocumentStore::Config _storeConfig; + const ThreadingServiceConfig _threading_service_config; + const AllocConfig _alloc_config; + SP _orig; + bool _delayedAttributeAspects; template <typename T> @@ -154,9 +154,9 @@ private: public: DocumentDBConfig(int64_t generation, const RankProfilesConfigSP &rankProfiles, - const RankingConstants::SP &rankingConstants, - const RankingExpressions::SP &rankingExpressions, - const OnnxModels::SP &onnxModels, + const std::shared_ptr<const RankingConstants> &rankingConstants, + const std::shared_ptr<const RankingExpressions> &rankingExpressions, + const std::shared_ptr<const OnnxModels> &onnxModels, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, const SummaryConfigSP &summary, @@ -193,9 +193,9 @@ public: const JuniperrcConfig &getJuniperrcConfig() const { return *_juniperrc; } const DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } const RankProfilesConfigSP &getRankProfilesConfigSP() const { return _rankProfiles; } - const RankingConstants::SP &getRankingConstantsSP() const { return _rankingConstants; } - const RankingExpressions::SP &getRankingExpressionsSP() const { return _rankingExpressions; } - const OnnxModels::SP &getOnnxModelsSP() const { return _onnxModels; } + const std::shared_ptr<const RankingConstants> &getRankingConstantsSP() const { return _rankingConstants; } + const std::shared_ptr<const RankingExpressions> &getRankingExpressionsSP() const { return _rankingExpressions; } + const std::shared_ptr<const OnnxModels> &getOnnxModelsSP() const { return _onnxModels; } const IndexschemaConfigSP &getIndexschemaConfigSP() const { return _indexschema; } const AttributesConfigSP &getAttributesConfigSP() const { return _attributes; } const SummaryConfigSP &getSummaryConfigSP() const { return _summary; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 96f6c87890d..35dd59a91d3 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -12,21 +12,19 @@ #include <vespa/config-ranking-expressions.h> #include <vespa/config-summary.h> #include <vespa/config/common/exceptions.h> -#include <vespa/config/file_acquirer/file_acquirer.h> #include <vespa/config/common/configcontext.h> #include <vespa/config/retriever/configretriever.h> #include <vespa/config/helper/legacy.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> +#include <vespa/searchlib/fef/ranking_assets_builder.h> #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchsummary/config/config-juniperrc.h> -#include <vespa/vespalib/time/time_box.h> -#include <vespa/vespalib/util/stringfmt.h> -#include <vespa/vespalib/util/time.h> #include <vespa/config/retriever/configsnapshot.hpp> #include <thread> #include <cassert> +#include <cinttypes> #include <vespa/log/log.h> LOG_SETUP(".proton.server.documentdbconfigmanager"); @@ -39,6 +37,7 @@ using namespace vespa::config::search; using document::DocumentTypeRepo; using search::TuneFileDocumentDB; using search::fef::OnnxModels; +using search::fef::RankingAssetsBuilder; using search::fef::RankingConstants; using search::fef::RankingExpressions; using search::index::Schema; @@ -52,8 +51,6 @@ using std::make_shared; using std::make_unique; using vespalib::datastore::CompactionStrategy; -using vespalib::make_string_short::fmt; - namespace proton { const ConfigKeySet @@ -238,26 +235,6 @@ build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& do distribution_config.redundancy, distribution_config.searchablecopies); } -vespalib::string -resolve_file(config::RpcFileAcquirer &fileAcquirer, vespalib::TimeBox &timeBox, - const vespalib::string &desc, const vespalib::string &fileref) -{ - vespalib::string filePath; - LOG(debug, "Waiting for file acquirer (%s, ref='%s')", desc.c_str(), fileref.c_str()); - while (timeBox.hasTimeLeft() && (filePath == "")) { - filePath = fileAcquirer.wait_for(fileref, timeBox.timeLeft()); - if (filePath == "") { - std::this_thread::sleep_for(100ms); - } - } - LOG(debug, "Got file path from file acquirer: '%s' (%s, ref='%s')", filePath.c_str(), desc.c_str(), fileref.c_str()); - if (filePath == "") { - throw config::ConfigTimeoutException(fmt("could not get file path from file acquirer for %s (ref=%s)", - desc.c_str(), fileref.c_str())); - } - return filePath; -} - } void @@ -275,13 +252,12 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot DocumentDBConfig::SP current = _pendingConfigSnapshot; RankProfilesConfigSP newRankProfilesConfig; - search::fef::RankingConstants::SP newRankingConstants; - search::fef::RankingExpressions::SP newRankingExpressions; - search::fef::OnnxModels::SP newOnnxModels; + std::shared_ptr<const RankingConstants> newRankingConstants; + std::shared_ptr<const RankingExpressions> newRankingExpressions; + std::shared_ptr<const OnnxModels> newOnnxModels; IndexschemaConfigSP newIndexschemaConfig; MaintenanceConfigSP oldMaintenanceConfig; MaintenanceConfigSP newMaintenanceConfig; - constexpr vespalib::duration file_resolve_timeout = 60min; if (!_ignoreForwardedConfig) { if (!(_bootstrapConfig->getDocumenttypesConfigSP() && @@ -312,52 +288,21 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) { newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId); } - vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5); + RankingAssetsBuilder ranking_assets_builder(transport, _bootstrapConfig->getFiledistributorrpcConfig().connectionspec); if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) { RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP( snapshot.getConfig<RankingConstantsConfig>(_configId)); - const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec; - RankingConstants::Vector constants; - if (spec != "") { - config::RpcFileAcquirer fileAcquirer(transport, spec); - for (const RankingConstantsConfig::Constant &rc : newRankingConstantsConfig->constant) { - auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str()); - vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); - constants.emplace_back(rc.name, rc.type, filePath); - } - } - newRankingConstants = std::make_shared<RankingConstants>(constants); + newRankingConstants = ranking_assets_builder.build(*newRankingConstantsConfig); } if (snapshot.isChanged<RankingExpressionsConfig>(_configId, currentGeneration)) { RankingExpressionsConfigSP newRankingExpressionsConfig = RankingExpressionsConfigSP( snapshot.getConfig<RankingExpressionsConfig>(_configId)); - const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec; - RankingExpressions expressions; - if (spec != "") { - config::RpcFileAcquirer fileAcquirer(transport, spec); - for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) { - auto desc = fmt("name='%s'", rc.name.c_str()); - vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); - expressions.add(rc.name, filePath); - } - } - newRankingExpressions = std::make_shared<RankingExpressions>(std::move(expressions)); + newRankingExpressions = ranking_assets_builder.build(*newRankingExpressionsConfig); } if (snapshot.isChanged<OnnxModelsConfig>(_configId, currentGeneration)) { OnnxModelsConfigSP newOnnxModelsConfig = OnnxModelsConfigSP( snapshot.getConfig<OnnxModelsConfig>(_configId)); - const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec; - OnnxModels::Vector models; - if (spec != "") { - config::RpcFileAcquirer fileAcquirer(transport, spec); - for (const OnnxModelsConfig::Model &rc : newOnnxModelsConfig->model) { - auto desc = fmt("name='%s'", rc.name.c_str()); - vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); - models.emplace_back(rc.name, filePath); - OnnxModels::configure(rc, models.back()); - } - } - newOnnxModels = std::make_shared<OnnxModels>(std::move(models)); + newOnnxModels = ranking_assets_builder.build(*newOnnxModelsConfig); } if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) { newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId); diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 3f84128c7f9..c16cd11c3f3 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -11,11 +11,14 @@ namespace proton::test { */ class DocumentDBConfigBuilder { private: + using OnnxModels = DocumentDBConfig::OnnxModels; + using RankingConstants = DocumentDBConfig::RankingConstants; + using RankingExpressions = DocumentDBConfig::RankingExpressions; int64_t _generation; DocumentDBConfig::RankProfilesConfigSP _rankProfiles; - DocumentDBConfig::RankingConstants::SP _rankingConstants; - DocumentDBConfig::RankingExpressions::SP _rankingExpressions; - DocumentDBConfig::OnnxModels::SP _onnxModels; + std::shared_ptr<const RankingConstants> _rankingConstants; + std::shared_ptr<const RankingExpressions> _rankingExpressions; + std::shared_ptr<const OnnxModels> _onnxModels; DocumentDBConfig::IndexschemaConfigSP _indexschema; DocumentDBConfig::AttributesConfigSP _attributes; DocumentDBConfig::SummaryConfigSP _summary; @@ -53,15 +56,15 @@ public: _attributes = attributes_in; return *this; } - DocumentDBConfigBuilder &rankingConstants(const DocumentDBConfig::RankingConstants::SP &rankingConstants_in) { + DocumentDBConfigBuilder &rankingConstants(const std::shared_ptr<const RankingConstants> &rankingConstants_in) { _rankingConstants = rankingConstants_in; return *this; } - DocumentDBConfigBuilder &rankingExpressions(const DocumentDBConfig::RankingExpressions::SP &rankingExpressions_in) { + DocumentDBConfigBuilder &rankingExpressions(const std::shared_ptr<const RankingExpressions> &rankingExpressions_in) { _rankingExpressions = rankingExpressions_in; return *this; } - DocumentDBConfigBuilder &onnxModels(const DocumentDBConfig::OnnxModels::SP &onnxModels_in) { + DocumentDBConfigBuilder &onnxModels(const std::shared_ptr<const OnnxModels> &onnxModels_in) { _onnxModels = onnxModels_in; return *this; } |