Add RankingAssetsBuilder.

6 files changed, 137 insertions, 3 deletions

diff --git a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt
index 2ea9349861b..299bc58a38e 100644
--- a/searchlib/src/vespa/searchlib/fef/CMakeLists.txt
+++ b/searchlib/src/vespa/searchlib/fef/CMakeLists.txt
@@ -30,6 +30,7 @@ vespa_add_library(searchlib_fef OBJECT
     query_value.cpp
     queryproperties.cpp
     rank_program.cpp
+    ranking_assets_builder.cpp
     ranking_assets_repo.cpp
     ranking_constants.cpp
     ranking_expressions.cpp
diff --git a/searchlib/src/vespa/searchlib/fef/onnx_models.h b/searchlib/src/vespa/searchlib/fef/onnx_models.h
index c6c4a2b3fe9..cdf92cb5d69 100644
--- a/searchlib/src/vespa/searchlib/fef/onnx_models.h
+++ b/searchlib/src/vespa/searchlib/fef/onnx_models.h
@@ -25,7 +25,6 @@ private:
     Map _models;
 
 public:
-    using SP = std::shared_ptr<OnnxModels>;
     OnnxModels();
     OnnxModels(Vector models);
     OnnxModels(OnnxModels &&) noexcept;
diff --git a/searchlib/src/vespa/searchlib/fef/ranking_assets_builder.cpp b/searchlib/src/vespa/searchlib/fef/ranking_assets_builder.cpp
new file mode 100644
index 00000000000..a31db7338c0
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/fef/ranking_assets_builder.cpp
@@ -0,0 +1,99 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#include "ranking_assets_builder.h"
+#include "onnx_models.h"
+#include "ranking_constants.h"
+#include "ranking_expressions.h"
+#include <vespa/config/common/exceptions.h>
+#include <vespa/config/file_acquirer/file_acquirer.h>
+#include <vespa/vespalib/util/stringfmt.h>
+#include <vespa/vespalib/util/time.h>
+#include <thread>
+
+#include <vespa/log/log.h>
+LOG_SETUP(".fef.ranking_assets_builder");
+
+
+using vespa::config::search::core::OnnxModelsConfig;
+using vespa::config::search::core::RankingConstantsConfig;
+using vespa::config::search::core::RankingExpressionsConfig;
+using vespalib::make_string_short::fmt;
+
+namespace search::fef {
+
+constexpr vespalib::duration file_resolve_timeout = 60min;
+
+RankingAssetsBuilder::RankingAssetsBuilder(FNET_Transport& transport, const vespalib::string& file_distributor_connection_spec)
+    : _file_acquirer(),
+      _time_box(vespalib::to_s(file_resolve_timeout), 5)
+{
+    if (file_distributor_connection_spec != "") {
+        _file_acquirer = std::make_unique<config::RpcFileAcquirer>(transport, file_distributor_connection_spec);
+    }
+}
+
+RankingAssetsBuilder::~RankingAssetsBuilder() = default;
+
+vespalib::string
+RankingAssetsBuilder::resolve_file(const vespalib::string& desc, const vespalib::string& fileref)
+{
+    vespalib::string file_path;
+    LOG(debug, "Waiting for file acquirer (%s, ref='%s')", desc.c_str(), fileref.c_str());
+    while (_time_box.hasTimeLeft() && (file_path == "")) {
+        file_path = _file_acquirer->wait_for(fileref, _time_box.timeLeft());
+        if (file_path == "") {
+            std::this_thread::sleep_for(100ms);
+        }
+    }
+    LOG(debug, "Got file path from file acquirer: '%s' (%s, ref='%s')", file_path.c_str(), desc.c_str(), fileref.c_str());
+    if (file_path == "") {
+        throw config::ConfigTimeoutException(fmt("could not get file path from file acquirer for %s (ref=%s)",
+                                                 desc.c_str(), fileref.c_str()));
+    }
+    return file_path;
+}
+
+std::shared_ptr<const OnnxModels>
+RankingAssetsBuilder::build(const OnnxModelsConfig& config)
+{
+    OnnxModels::Vector models;
+    if (_file_acquirer) {
+        for (const OnnxModelsConfig::Model &rc : config.model) {
+            auto desc = fmt("name='%s'", rc.name.c_str());
+            vespalib::string file_path = resolve_file(desc, rc.fileref);
+            models.emplace_back(rc.name, file_path);
+            OnnxModels::configure(rc, models.back());
+        }
+    }
+    return std::make_shared<OnnxModels>(std::move(models));
+}
+
+std::shared_ptr<const RankingConstants>
+RankingAssetsBuilder::build(const RankingConstantsConfig& config)
+{
+    RankingConstants::Vector constants;
+    if (_file_acquirer) {
+        for (const RankingConstantsConfig::Constant &rc : config.constant) {
+            auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str());
+            vespalib::string file_path = resolve_file(desc, rc.fileref);
+            constants.emplace_back(rc.name, rc.type, file_path);
+        }
+    }
+    return std::make_shared<RankingConstants>(constants);
+}
+
+std::shared_ptr<const RankingExpressions>
+RankingAssetsBuilder::build(const RankingExpressionsConfig& config)
+{
+    RankingExpressions expressions;
+    if (_file_acquirer) {
+        for (const RankingExpressionsConfig::Expression &rc : config.expression) {
+            auto desc = fmt("name='%s'", rc.name.c_str());
+            vespalib::string filePath = resolve_file(desc, rc.fileref);
+            expressions.add(rc.name, filePath);
+        }
+    }
+    return std::make_shared<RankingExpressions>(std::move(expressions));
+}
+
+}
diff --git a/searchlib/src/vespa/searchlib/fef/ranking_assets_builder.h b/searchlib/src/vespa/searchlib/fef/ranking_assets_builder.h
new file mode 100644
index 00000000000..b37aad8b905
--- /dev/null
+++ b/searchlib/src/vespa/searchlib/fef/ranking_assets_builder.h
@@ -0,0 +1,37 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+
+#pragma once
+
+#include <vespa/config-onnx-models.h>
+#include <vespa/config-ranking-constants.h>
+#include <vespa/config-ranking-expressions.h>
+#include <vespa/vespalib/stllike/string.h>
+#include <vespa/vespalib/time/time_box.h>
+
+class FNET_Transport;
+
+namespace config { struct FileAcquirer; }
+
+namespace search::fef {
+
+class OnnxModels;
+class RankingConstants;
+class RankingExpressions;
+
+/*
+ * Builder class for ranking assets (OnnxModels, RankingConstants, RankingExpressions).
+ */
+class RankingAssetsBuilder {
+    std::unique_ptr<config::FileAcquirer> _file_acquirer;
+    vespalib::TimeBox                     _time_box;
+
+    vespalib::string resolve_file(const vespalib::string& desc, const vespalib::string& fileref);
+public:
+    RankingAssetsBuilder(FNET_Transport& transport, const vespalib::string& file_distributor_connection_spec);
+    ~RankingAssetsBuilder();
+    std::shared_ptr<const OnnxModels> build(const vespa::config::search::core::OnnxModelsConfig& config);
+    std::shared_ptr<const RankingConstants> build(const vespa::config::search::core::RankingConstantsConfig& config);
+    std::shared_ptr<const RankingExpressions> build(const vespa::config::search::core::RankingExpressionsConfig& config);
+};
+
+}
diff --git a/searchlib/src/vespa/searchlib/fef/ranking_constants.h b/searchlib/src/vespa/searchlib/fef/ranking_constants.h
index 56c124986ba..1706719dbdb 100644
--- a/searchlib/src/vespa/searchlib/fef/ranking_constants.h
+++ b/searchlib/src/vespa/searchlib/fef/ranking_constants.h
@@ -33,7 +33,6 @@ private:
     Map _constants;
 
 public:
-    using SP = std::shared_ptr<RankingConstants>;
     RankingConstants();
     RankingConstants(RankingConstants &&) noexcept;
     RankingConstants & operator =(RankingConstants &&) = delete;
diff --git a/searchlib/src/vespa/searchlib/fef/ranking_expressions.h b/searchlib/src/vespa/searchlib/fef/ranking_expressions.h
index 317015938e6..8776cf5d54c 100644
--- a/searchlib/src/vespa/searchlib/fef/ranking_expressions.h
+++ b/searchlib/src/vespa/searchlib/fef/ranking_expressions.h
@@ -19,7 +19,6 @@ private:
     std::map<vespalib::string,vespalib::string> _expressions;
 
 public:
-    using SP = std::shared_ptr<RankingExpressions>;
     RankingExpressions();
     RankingExpressions(RankingExpressions &&rhs) noexcept;
     RankingExpressions & operator=(RankingExpressions &&rhs) = delete;