add streaming-mode handling

4 files changed, 67 insertions, 20 deletions

diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
index 4411babeb10..13e4092c2ad 100644
--- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
+++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt
@@ -4,6 +4,7 @@ vespa_add_library(searchcore_verify_ranksetup
     verify_ranksetup.cpp
     INSTALL lib64
     DEPENDS
+    streamingvisitors
     searchcore_matching
     searchcore_documentmetastore
 )
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
index 513290cc4d1..7f535fd4852 100644
--- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp
@@ -22,10 +22,16 @@
 #include <vespa/searchlib/fef/onnx_models.h>
 #include <vespa/searchlib/fef/ranking_expressions.h>
 #include <vespa/searchlib/fef/test/plugin/setup.h>
+
+#include <vespa/searchvisitor/indexenvironment.h>
+#include <vespa/searchvisitor/rankmanager.h>
+#include <vespa/vsm/config/config-vsmfields.h>
+
 #include <vespa/config/subscription/configsubscriber.hpp>
 #include <vespa/vespalib/util/stringfmt.h>
 #include <vespa/vespalib/stllike/asciistream.h>
 #include <optional>
+#include <functional>
 
 using config::ConfigContext;
 using config::ConfigHandle;
@@ -43,6 +49,7 @@ using vespa::config::search::core::RankingConstantsConfig;
 using vespa::config::search::core::RankingExpressionsConfig;
 using vespa::config::search::core::OnnxModelsConfig;
 using vespa::config::search::core::VerifyRanksetupConfig;
+using vespa::config::search::vsm::VsmfieldsConfig;
 using vespalib::eval::BadConstantValue;
 using vespalib::eval::ConstantValue;
 using vespalib::eval::FastValueBuilderFactory;
@@ -99,11 +106,12 @@ class VerifyRankSetup
 {
 private:
     std::vector<search::fef::Message> _messages;
-    bool verify(const search::index::Schema &schema,
-                const search::fef::Properties &props,
-                const IRankingAssetsRepo &repo);
+    SearchMode _searchMode;
+
+    bool verifyIndexEnv(const search::fef::IIndexEnvironment &indexEnv);
 
     bool verifyConfig(const VerifyRanksetupConfig &myCfg,
+                      const VsmfieldsConfig &vsmFieldsCcfg,
                       const RankProfilesConfig &rankCfg,
                       const IndexschemaConfig &schemaCfg,
                       const AttributesConfig &attributeCfg,
@@ -112,7 +120,7 @@ private:
                       const OnnxModelsConfig &modelsCfg);
 
 public:
-    VerifyRankSetup();
+    explicit VerifyRankSetup(SearchMode mode);
     ~VerifyRankSetup();
     [[nodiscard]] const std::vector<search::fef::Message> & getMessages() const { return _messages; }
     bool verify(const std::string & configId);
@@ -140,7 +148,9 @@ DummyRankingAssetsRepo::DummyRankingAssetsRepo(const RankingConstantsConfig &cfg
       _expressions(std::move(expressions)),
       _onnxModels(std::move(onnxModels))
 {}
+
 DummyRankingAssetsRepo::~DummyRankingAssetsRepo() = default;
+
 vespalib::eval::ConstantValue::UP
 DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const {
     for (const auto &entry: cfg.constant) {
@@ -156,18 +166,15 @@ DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const {
     return {};
 }
 
-VerifyRankSetup::VerifyRankSetup()
-    : _messages()
+VerifyRankSetup::VerifyRankSetup(SearchMode mode)
+    : _messages(),
+      _searchMode(mode)
 { }
 
 VerifyRankSetup::~VerifyRankSetup() = default;
 
 bool
-VerifyRankSetup::verify(const search::index::Schema &schema,
-                        const search::fef::Properties &props,
-                        const IRankingAssetsRepo &repo)
-{
-    proton::matching::IndexEnvironment indexEnv(0, schema, props, repo);
+VerifyRankSetup::verifyIndexEnv(const search::fef::IIndexEnvironment &indexEnv) {
     search::fef::BlueprintFactory factory;
     search::features::setup_search_features(factory);
     search::fef::test::setup_fef_test_plugin(factory);
@@ -195,6 +202,7 @@ VerifyRankSetup::verify(const search::index::Schema &schema,
 
 bool
 VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg,
+                              const VsmfieldsConfig &vsmFieldsCfg,
                               const RankProfilesConfig &rankCfg,
                               const IndexschemaConfig &schemaCfg,
                               const AttributesConfig &attributeCfg,
@@ -203,17 +211,38 @@ VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg,
                               const OnnxModelsConfig &modelsCfg)
 {
     bool ok = true;
+    auto repo = std::make_shared<DummyRankingAssetsRepo>(constantsCfg,
+                                                         make_expressions(expressionsCfg, myCfg, _messages),
+                                                         make_models(modelsCfg, myCfg, _messages));
+
+    using IndexEnvFactory = std::function<std::unique_ptr<search::fef::IIndexEnvironment>(const search::fef::Properties &)>;
+    IndexEnvFactory factory;
+    streaming::IndexEnvPrototype streamingProto;
     search::index::Schema schema;
-    search::index::SchemaBuilder::build(schemaCfg, schema);
-    search::index::SchemaBuilder::build(attributeCfg, schema);
-    DummyRankingAssetsRepo repo(constantsCfg, make_expressions(expressionsCfg, myCfg, _messages),
-                                make_models(modelsCfg, myCfg, _messages));
+    if (_searchMode == SearchMode::STREAMING) {
+        streamingProto.set_ranking_assets_repo(repo);
+        streamingProto.detectFields(vsmFieldsCfg);
+        factory = [&](const search::fef::Properties &properties)
+                  {
+                      auto indexEnv = streamingProto.clone();
+                      indexEnv->getProperties().import(properties);
+                      return indexEnv;
+                  };
+    } else {
+        search::index::SchemaBuilder::build(schemaCfg, schema);
+        search::index::SchemaBuilder::build(attributeCfg, schema);
+        factory = [&](const search::fef::Properties &properties)
+                  {
+                      return std::make_unique<proton::matching::IndexEnvironment>(0, schema, properties, *repo);
+                  };
+    }
     for(const auto & profile : rankCfg.rankprofile) {
         search::fef::Properties properties;
         for(const auto & j : profile.fef.property) {
             properties.add(j.name, j.value);
         }
-        if (verify(schema, properties, repo)) {
+        auto indexEnvP = factory(properties);
+        if (verifyIndexEnv(*indexEnvP)) {
             _messages.emplace_back(search::fef::Level::INFO,
                                    fmt("rank profile '%s': pass", profile.name.c_str()));
         } else {
@@ -241,8 +270,17 @@ VerifyRankSetup::verify(const std::string & configid)
         ConfigHandle<RankingExpressionsConfig>::UP expressionsHandle = subscriber.subscribe<RankingExpressionsConfig>(cfgId);
         ConfigHandle<OnnxModelsConfig>::UP modelsHandle = subscriber.subscribe<OnnxModelsConfig>(cfgId);
 
+        std::unique_ptr<VsmfieldsConfig> vsmFieldsCfg = std::make_unique<VsmfieldsConfig>();
+        ConfigHandle<VsmfieldsConfig>::UP vsmFieldsHandle;
+        if (_searchMode == SearchMode::STREAMING) {
+            vsmFieldsHandle = subscriber.subscribe<VsmfieldsConfig>(cfgId);
+        }
         subscriber.nextConfig();
+        if (_searchMode == SearchMode::STREAMING) {
+            vsmFieldsCfg = vsmFieldsHandle->getConfig();
+        }
         ok = verifyConfig(*myHandle->getConfig(),
+                          *vsmFieldsCfg,
                           *rankHandle->getConfig(),
                           *schemaHandle->getConfig(),
                           *attributesHandle->getConfig(),
@@ -260,8 +298,8 @@ VerifyRankSetup::verify(const std::string & configid)
 }
 
 std::pair<bool, std::vector<search::fef::Message>>
-verifyRankSetup(const char * configId) {
-    VerifyRankSetup verifier;
+verifyRankSetup(const char * configId, SearchMode mode) {
+    VerifyRankSetup verifier{mode};
     bool ok = verifier.verify(configId);
 
     return {ok, verifier.getMessages()};
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h
index 8e77bdd51fb..4b18805fe0c 100644
--- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h
@@ -4,4 +4,6 @@
 
 #include <vespa/searchlib/fef/verify_feature.h>
 
-std::pair<bool, std::vector<search::fef::Message>> verifyRankSetup(const char * configId);
+enum class SearchMode { INDEXED, STREAMING };
+
+std::pair<bool, std::vector<search::fef::Message>> verifyRankSetup(const char * configId, SearchMode mode);
diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp
index 4d2c657fc70..e179685b55b 100644
--- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp
+++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp
@@ -34,14 +34,20 @@ toLogLevel(search::fef::Level level) {
     abort();
 }
 }
+
 int
 App::main(int argc, char **argv)
 {
+    SearchMode mode = SearchMode::INDEXED;
+    if (argc == 3 && (strcmp("-S", argv[2]) == 0)) {
+        mode = SearchMode::STREAMING;
+        --argc;
+    }
     if (argc != 2) {
         return usage();
     }
 
-    auto [ok, messages] = verifyRankSetup(argv[1]);
+    auto [ok, messages] = verifyRankSetup(argv[1], mode);
 
     for (const auto & msg : messages) {
         VLOG(toLogLevel(msg.first), "%s", msg.second.c_str());