Add RankingAssetsBuilder.

5 files changed, 52 insertions, 104 deletions

diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
index d8dafd1c39e..88ccec80b8a 100644
--- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
+++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp
@@ -58,17 +58,17 @@ struct DBConfigFixture {
         return DocumentDBConfig::build_schema(_attributesBuilder, _indexschemaBuilder);
     }
 
-    static RankingConstants::SP buildRankingConstants()
+    static std::shared_ptr<const RankingConstants> buildRankingConstants()
     {
         return std::make_shared<RankingConstants>();
     }
 
-    static RankingExpressions::SP buildRankingExpressions()
+    static std::shared_ptr<const RankingExpressions> buildRankingExpressions()
     {
         return std::make_shared<RankingExpressions>();
     }
 
-    static OnnxModels::SP buildOnnxModels()
+    static std::shared_ptr<const OnnxModels> buildOnnxModels()
     {
         return std::make_shared<OnnxModels>();
     }
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
index 00d9c194b9e..cc8bc2c8b96 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp
@@ -55,9 +55,9 @@ DocumentDBConfig::ComparisonResult::ComparisonResult()
 DocumentDBConfig::DocumentDBConfig(
                int64_t generation,
                const RankProfilesConfigSP &rankProfiles,
-               const RankingConstants::SP &rankingConstants,
-               const RankingExpressions::SP &rankingExpressions,
-               const OnnxModels::SP &onnxModels,
+               const std::shared_ptr<const RankingConstants> &rankingConstants,
+               const std::shared_ptr<const RankingExpressions> &rankingExpressions,
+               const std::shared_ptr<const OnnxModels> &onnxModels,
                const IndexschemaConfigSP &indexschema,
                const AttributesConfigSP &attributes,
                const SummaryConfigSP &summary,
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
index 11f23cf0b66..848e68ce0f2 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h
@@ -111,28 +111,28 @@ public:
     using ImportedFieldsConfigSP = std::shared_ptr<ImportedFieldsConfig>;
 
 private:
-    vespalib::string                 _configId;
-    vespalib::string                 _docTypeName;
-    int64_t                          _generation;
-    RankProfilesConfigSP             _rankProfiles;
-    RankingConstants::SP             _rankingConstants;
-    RankingExpressions::SP           _rankingExpressions;
-    OnnxModels::SP                   _onnxModels;
-    IndexschemaConfigSP              _indexschema;
-    AttributesConfigSP               _attributes;
-    SummaryConfigSP                  _summary;
-    JuniperrcConfigSP                _juniperrc;
-    DocumenttypesConfigSP            _documenttypes;
+    vespalib::string                          _configId;
+    vespalib::string                          _docTypeName;
+    int64_t                                   _generation;
+    RankProfilesConfigSP                      _rankProfiles;
+    std::shared_ptr<const RankingConstants>   _rankingConstants;
+    std::shared_ptr<const RankingExpressions> _rankingExpressions;
+    std::shared_ptr<const OnnxModels>         _onnxModels;
+    IndexschemaConfigSP                       _indexschema;
+    AttributesConfigSP                        _attributes;
+    SummaryConfigSP                           _summary;
+    JuniperrcConfigSP                         _juniperrc;
+    DocumenttypesConfigSP                     _documenttypes;
     std::shared_ptr<const document::DocumentTypeRepo>   _repo;
-    ImportedFieldsConfigSP           _importedFields;
-    search::TuneFileDocumentDB::SP   _tuneFileDocumentDB;
-    search::index::Schema::SP        _schema;
-    MaintenanceConfigSP              _maintenance;
-    search::LogDocumentStore::Config _storeConfig;
-    const ThreadingServiceConfig     _threading_service_config;
-    const AllocConfig                _alloc_config;
-    SP                               _orig;
-    bool                             _delayedAttributeAspects;
+    ImportedFieldsConfigSP                    _importedFields;
+    search::TuneFileDocumentDB::SP            _tuneFileDocumentDB;
+    search::index::Schema::SP                 _schema;
+    MaintenanceConfigSP                       _maintenance;
+    search::LogDocumentStore::Config          _storeConfig;
+    const ThreadingServiceConfig              _threading_service_config;
+    const AllocConfig                         _alloc_config;
+    SP                                        _orig;
+    bool                                      _delayedAttributeAspects;
 
 
     template <typename T>
@@ -154,9 +154,9 @@ private:
 public:
     DocumentDBConfig(int64_t generation,
                      const RankProfilesConfigSP &rankProfiles,
-                     const RankingConstants::SP &rankingConstants,
-                     const RankingExpressions::SP &rankingExpressions,
-                     const OnnxModels::SP &onnxModels,
+                     const std::shared_ptr<const RankingConstants> &rankingConstants,
+                     const std::shared_ptr<const RankingExpressions> &rankingExpressions,
+                     const std::shared_ptr<const OnnxModels> &onnxModels,
                      const IndexschemaConfigSP &indexschema,
                      const AttributesConfigSP &attributes,
                      const SummaryConfigSP &summary,
@@ -193,9 +193,9 @@ public:
     const JuniperrcConfig &getJuniperrcConfig() const { return *_juniperrc; }
     const DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; }
     const RankProfilesConfigSP &getRankProfilesConfigSP() const { return _rankProfiles; }
-    const RankingConstants::SP &getRankingConstantsSP() const { return _rankingConstants; }
-    const RankingExpressions::SP &getRankingExpressionsSP() const { return _rankingExpressions; }
-    const OnnxModels::SP &getOnnxModelsSP() const { return _onnxModels; }
+    const std::shared_ptr<const RankingConstants> &getRankingConstantsSP() const { return _rankingConstants; }
+    const std::shared_ptr<const RankingExpressions> &getRankingExpressionsSP() const { return _rankingExpressions; }
+    const std::shared_ptr<const OnnxModels> &getOnnxModelsSP() const { return _onnxModels; }
     const IndexschemaConfigSP &getIndexschemaConfigSP() const { return _indexschema; }
     const AttributesConfigSP &getAttributesConfigSP() const { return _attributes; }
     const SummaryConfigSP &getSummaryConfigSP() const { return _summary; }
diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
index 96f6c87890d..35dd59a91d3 100644
--- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
+++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp
@@ -12,21 +12,19 @@
 #include <vespa/config-ranking-expressions.h>
 #include <vespa/config-summary.h>
 #include <vespa/config/common/exceptions.h>
-#include <vespa/config/file_acquirer/file_acquirer.h>
 #include <vespa/config/common/configcontext.h>
 #include <vespa/config/retriever/configretriever.h>
 #include <vespa/config/helper/legacy.h>
 #include <vespa/searchcommon/common/schemaconfigurer.h>
 #include <vespa/searchcore/proton/common/alloc_config.h>
 #include <vespa/searchcore/proton/common/hw_info.h>
+#include <vespa/searchlib/fef/ranking_assets_builder.h>
 #include <vespa/searchlib/index/schemautil.h>
 #include <vespa/searchsummary/config/config-juniperrc.h>
-#include <vespa/vespalib/time/time_box.h>
-#include <vespa/vespalib/util/stringfmt.h>
-#include <vespa/vespalib/util/time.h>
 #include <vespa/config/retriever/configsnapshot.hpp>
 #include <thread>
 #include <cassert>
+#include <cinttypes>
 
 #include <vespa/log/log.h>
 LOG_SETUP(".proton.server.documentdbconfigmanager");
@@ -39,6 +37,7 @@ using namespace vespa::config::search;
 using document::DocumentTypeRepo;
 using search::TuneFileDocumentDB;
 using search::fef::OnnxModels;
+using search::fef::RankingAssetsBuilder;
 using search::fef::RankingConstants;
 using search::fef::RankingExpressions;
 using search::index::Schema;
@@ -52,8 +51,6 @@ using std::make_shared;
 using std::make_unique;
 using vespalib::datastore::CompactionStrategy;
 
-using vespalib::make_string_short::fmt;
-
 namespace proton {
 
 const ConfigKeySet
@@ -238,26 +235,6 @@ build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& do
                        distribution_config.redundancy, distribution_config.searchablecopies);
 }
 
-vespalib::string
-resolve_file(config::RpcFileAcquirer &fileAcquirer, vespalib::TimeBox &timeBox,
-             const vespalib::string &desc, const vespalib::string &fileref)
-{
-    vespalib::string filePath;
-    LOG(debug, "Waiting for file acquirer (%s, ref='%s')", desc.c_str(), fileref.c_str());
-    while (timeBox.hasTimeLeft() && (filePath == "")) {
-        filePath = fileAcquirer.wait_for(fileref, timeBox.timeLeft());
-        if (filePath == "") {
-            std::this_thread::sleep_for(100ms);
-        }
-    }
-    LOG(debug, "Got file path from file acquirer: '%s' (%s, ref='%s')", filePath.c_str(), desc.c_str(), fileref.c_str());
-    if (filePath == "") {
-        throw config::ConfigTimeoutException(fmt("could not get file path from file acquirer for %s (ref=%s)",
-                                                 desc.c_str(), fileref.c_str()));
-    }
-    return filePath;
-}
-
 }
 
 void
@@ -275,13 +252,12 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot
 
     DocumentDBConfig::SP current = _pendingConfigSnapshot;
     RankProfilesConfigSP newRankProfilesConfig;
-    search::fef::RankingConstants::SP newRankingConstants;
-    search::fef::RankingExpressions::SP newRankingExpressions;
-    search::fef::OnnxModels::SP newOnnxModels;
+    std::shared_ptr<const RankingConstants> newRankingConstants;
+    std::shared_ptr<const RankingExpressions> newRankingExpressions;
+    std::shared_ptr<const OnnxModels> newOnnxModels;
     IndexschemaConfigSP newIndexschemaConfig;
     MaintenanceConfigSP oldMaintenanceConfig;
     MaintenanceConfigSP newMaintenanceConfig;
-    constexpr vespalib::duration file_resolve_timeout = 60min;
 
     if (!_ignoreForwardedConfig) {
         if (!(_bootstrapConfig->getDocumenttypesConfigSP() &&
@@ -312,52 +288,21 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot
     if (snapshot.isChanged<RankProfilesConfig>(_configId, currentGeneration)) {
         newRankProfilesConfig = snapshot.getConfig<RankProfilesConfig>(_configId);
     }
-    vespalib::TimeBox timeBox(vespalib::to_s(file_resolve_timeout), 5);
+    RankingAssetsBuilder ranking_assets_builder(transport, _bootstrapConfig->getFiledistributorrpcConfig().connectionspec);
     if (snapshot.isChanged<RankingConstantsConfig>(_configId, currentGeneration)) {
         RankingConstantsConfigSP newRankingConstantsConfig = RankingConstantsConfigSP(
                 snapshot.getConfig<RankingConstantsConfig>(_configId));
-        const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec;
-        RankingConstants::Vector constants;
-        if (spec != "") {
-            config::RpcFileAcquirer fileAcquirer(transport, spec);
-            for (const RankingConstantsConfig::Constant &rc : newRankingConstantsConfig->constant) {
-                auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str());
-                vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref);
-                constants.emplace_back(rc.name, rc.type, filePath);
-            }
-        }
-        newRankingConstants = std::make_shared<RankingConstants>(constants);
+        newRankingConstants = ranking_assets_builder.build(*newRankingConstantsConfig);
     }
     if (snapshot.isChanged<RankingExpressionsConfig>(_configId, currentGeneration)) {
         RankingExpressionsConfigSP newRankingExpressionsConfig = RankingExpressionsConfigSP(
             snapshot.getConfig<RankingExpressionsConfig>(_configId));
-        const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec;
-        RankingExpressions expressions;
-        if (spec != "") {
-            config::RpcFileAcquirer fileAcquirer(transport, spec);
-            for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) {
-                auto desc = fmt("name='%s'", rc.name.c_str());
-                vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref);
-                expressions.add(rc.name, filePath);
-            }
-        }
-        newRankingExpressions = std::make_shared<RankingExpressions>(std::move(expressions));
+        newRankingExpressions = ranking_assets_builder.build(*newRankingExpressionsConfig);
     }
     if (snapshot.isChanged<OnnxModelsConfig>(_configId, currentGeneration)) {
         OnnxModelsConfigSP newOnnxModelsConfig = OnnxModelsConfigSP(
                 snapshot.getConfig<OnnxModelsConfig>(_configId));
-        const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec;
-        OnnxModels::Vector models;
-        if (spec != "") {
-            config::RpcFileAcquirer fileAcquirer(transport, spec);
-            for (const OnnxModelsConfig::Model &rc : newOnnxModelsConfig->model) {
-                auto desc = fmt("name='%s'", rc.name.c_str());
-                vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref);
-                models.emplace_back(rc.name, filePath);
-                OnnxModels::configure(rc, models.back());
-            }
-        }
-        newOnnxModels = std::make_shared<OnnxModels>(std::move(models));
+        newOnnxModels = ranking_assets_builder.build(*newOnnxModelsConfig);
     }
     if (snapshot.isChanged<IndexschemaConfig>(_configId, currentGeneration)) {
         newIndexschemaConfig = snapshot.getConfig<IndexschemaConfig>(_configId);
diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h
index 3f84128c7f9..c16cd11c3f3 100644
--- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h
+++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h
@@ -11,11 +11,14 @@ namespace proton::test {
  */
 class DocumentDBConfigBuilder {
 private:
+    using OnnxModels = DocumentDBConfig::OnnxModels;
+    using RankingConstants = DocumentDBConfig::RankingConstants;
+    using RankingExpressions = DocumentDBConfig::RankingExpressions;
     int64_t _generation;
     DocumentDBConfig::RankProfilesConfigSP _rankProfiles;
-    DocumentDBConfig::RankingConstants::SP _rankingConstants;
-    DocumentDBConfig::RankingExpressions::SP _rankingExpressions;
-    DocumentDBConfig::OnnxModels::SP _onnxModels;
+    std::shared_ptr<const RankingConstants> _rankingConstants;
+    std::shared_ptr<const RankingExpressions> _rankingExpressions;
+    std::shared_ptr<const OnnxModels> _onnxModels;
     DocumentDBConfig::IndexschemaConfigSP _indexschema;
     DocumentDBConfig::AttributesConfigSP _attributes;
     DocumentDBConfig::SummaryConfigSP _summary;
@@ -53,15 +56,15 @@ public:
         _attributes = attributes_in;
         return *this;
     }
-    DocumentDBConfigBuilder &rankingConstants(const DocumentDBConfig::RankingConstants::SP &rankingConstants_in) {
+    DocumentDBConfigBuilder &rankingConstants(const std::shared_ptr<const RankingConstants> &rankingConstants_in) {
         _rankingConstants = rankingConstants_in;
         return *this;
     }
-    DocumentDBConfigBuilder &rankingExpressions(const DocumentDBConfig::RankingExpressions::SP &rankingExpressions_in) {
+    DocumentDBConfigBuilder &rankingExpressions(const std::shared_ptr<const RankingExpressions> &rankingExpressions_in) {
         _rankingExpressions = rankingExpressions_in;
         return *this;
     }
-    DocumentDBConfigBuilder &onnxModels(const DocumentDBConfig::OnnxModels::SP &onnxModels_in) {
+    DocumentDBConfigBuilder &onnxModels(const std::shared_ptr<const OnnxModels> &onnxModels_in) {
         _onnxModels = onnxModels_in;
         return *this;
     }