From 445bb4e93365a98a58a5dfcaab40bbd50553dade Mon Sep 17 00:00:00 2001 From: Arne Juul Date: Wed, 17 Jan 2024 09:55:14 +0000 Subject: add streaming-mode handling --- .../src/apps/verify_ranksetup/CMakeLists.txt | 1 + .../src/apps/verify_ranksetup/verify_ranksetup.cpp | 74 ++++++++++++++++------ .../src/apps/verify_ranksetup/verify_ranksetup.h | 4 +- .../apps/verify_ranksetup/verify_ranksetup_app.cpp | 8 ++- 4 files changed, 67 insertions(+), 20 deletions(-) diff --git a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt index 4411babeb10..13e4092c2ad 100644 --- a/searchcore/src/apps/verify_ranksetup/CMakeLists.txt +++ b/searchcore/src/apps/verify_ranksetup/CMakeLists.txt @@ -4,6 +4,7 @@ vespa_add_library(searchcore_verify_ranksetup verify_ranksetup.cpp INSTALL lib64 DEPENDS + streamingvisitors searchcore_matching searchcore_documentmetastore ) diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 513290cc4d1..7f535fd4852 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -22,10 +22,16 @@ #include #include #include + +#include +#include +#include + #include #include #include #include +#include using config::ConfigContext; using config::ConfigHandle; @@ -43,6 +49,7 @@ using vespa::config::search::core::RankingConstantsConfig; using vespa::config::search::core::RankingExpressionsConfig; using vespa::config::search::core::OnnxModelsConfig; using vespa::config::search::core::VerifyRanksetupConfig; +using vespa::config::search::vsm::VsmfieldsConfig; using vespalib::eval::BadConstantValue; using vespalib::eval::ConstantValue; using vespalib::eval::FastValueBuilderFactory; @@ -99,11 +106,12 @@ class VerifyRankSetup { private: std::vector _messages; - bool verify(const search::index::Schema &schema, - const search::fef::Properties &props, - const IRankingAssetsRepo &repo); + SearchMode _searchMode; + + bool verifyIndexEnv(const search::fef::IIndexEnvironment &indexEnv); bool verifyConfig(const VerifyRanksetupConfig &myCfg, + const VsmfieldsConfig &vsmFieldsCcfg, const RankProfilesConfig &rankCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, @@ -112,7 +120,7 @@ private: const OnnxModelsConfig &modelsCfg); public: - VerifyRankSetup(); + explicit VerifyRankSetup(SearchMode mode); ~VerifyRankSetup(); [[nodiscard]] const std::vector & getMessages() const { return _messages; } bool verify(const std::string & configId); @@ -140,7 +148,9 @@ DummyRankingAssetsRepo::DummyRankingAssetsRepo(const RankingConstantsConfig &cfg _expressions(std::move(expressions)), _onnxModels(std::move(onnxModels)) {} + DummyRankingAssetsRepo::~DummyRankingAssetsRepo() = default; + vespalib::eval::ConstantValue::UP DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const { for (const auto &entry: cfg.constant) { @@ -156,18 +166,15 @@ DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const { return {}; } -VerifyRankSetup::VerifyRankSetup() - : _messages() +VerifyRankSetup::VerifyRankSetup(SearchMode mode) + : _messages(), + _searchMode(mode) { } VerifyRankSetup::~VerifyRankSetup() = default; bool -VerifyRankSetup::verify(const search::index::Schema &schema, - const search::fef::Properties &props, - const IRankingAssetsRepo &repo) -{ - proton::matching::IndexEnvironment indexEnv(0, schema, props, repo); +VerifyRankSetup::verifyIndexEnv(const search::fef::IIndexEnvironment &indexEnv) { search::fef::BlueprintFactory factory; search::features::setup_search_features(factory); search::fef::test::setup_fef_test_plugin(factory); @@ -195,6 +202,7 @@ VerifyRankSetup::verify(const search::index::Schema &schema, bool VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, + const VsmfieldsConfig &vsmFieldsCfg, const RankProfilesConfig &rankCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, @@ -203,17 +211,38 @@ VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, const OnnxModelsConfig &modelsCfg) { bool ok = true; + auto repo = std::make_shared(constantsCfg, + make_expressions(expressionsCfg, myCfg, _messages), + make_models(modelsCfg, myCfg, _messages)); + + using IndexEnvFactory = std::function(const search::fef::Properties &)>; + IndexEnvFactory factory; + streaming::IndexEnvPrototype streamingProto; search::index::Schema schema; - search::index::SchemaBuilder::build(schemaCfg, schema); - search::index::SchemaBuilder::build(attributeCfg, schema); - DummyRankingAssetsRepo repo(constantsCfg, make_expressions(expressionsCfg, myCfg, _messages), - make_models(modelsCfg, myCfg, _messages)); + if (_searchMode == SearchMode::STREAMING) { + streamingProto.set_ranking_assets_repo(repo); + streamingProto.detectFields(vsmFieldsCfg); + factory = [&](const search::fef::Properties &properties) + { + auto indexEnv = streamingProto.clone(); + indexEnv->getProperties().import(properties); + return indexEnv; + }; + } else { + search::index::SchemaBuilder::build(schemaCfg, schema); + search::index::SchemaBuilder::build(attributeCfg, schema); + factory = [&](const search::fef::Properties &properties) + { + return std::make_unique(0, schema, properties, *repo); + }; + } for(const auto & profile : rankCfg.rankprofile) { search::fef::Properties properties; for(const auto & j : profile.fef.property) { properties.add(j.name, j.value); } - if (verify(schema, properties, repo)) { + auto indexEnvP = factory(properties); + if (verifyIndexEnv(*indexEnvP)) { _messages.emplace_back(search::fef::Level::INFO, fmt("rank profile '%s': pass", profile.name.c_str())); } else { @@ -241,8 +270,17 @@ VerifyRankSetup::verify(const std::string & configid) ConfigHandle::UP expressionsHandle = subscriber.subscribe(cfgId); ConfigHandle::UP modelsHandle = subscriber.subscribe(cfgId); + std::unique_ptr vsmFieldsCfg = std::make_unique(); + ConfigHandle::UP vsmFieldsHandle; + if (_searchMode == SearchMode::STREAMING) { + vsmFieldsHandle = subscriber.subscribe(cfgId); + } subscriber.nextConfig(); + if (_searchMode == SearchMode::STREAMING) { + vsmFieldsCfg = vsmFieldsHandle->getConfig(); + } ok = verifyConfig(*myHandle->getConfig(), + *vsmFieldsCfg, *rankHandle->getConfig(), *schemaHandle->getConfig(), *attributesHandle->getConfig(), @@ -260,8 +298,8 @@ VerifyRankSetup::verify(const std::string & configid) } std::pair> -verifyRankSetup(const char * configId) { - VerifyRankSetup verifier; +verifyRankSetup(const char * configId, SearchMode mode) { + VerifyRankSetup verifier{mode}; bool ok = verifier.verify(configId); return {ok, verifier.getMessages()}; diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h index 8e77bdd51fb..4b18805fe0c 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.h @@ -4,4 +4,6 @@ #include -std::pair> verifyRankSetup(const char * configId); +enum class SearchMode { INDEXED, STREAMING }; + +std::pair> verifyRankSetup(const char * configId, SearchMode mode); diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp index 4d2c657fc70..e179685b55b 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup_app.cpp @@ -34,14 +34,20 @@ toLogLevel(search::fef::Level level) { abort(); } } + int App::main(int argc, char **argv) { + SearchMode mode = SearchMode::INDEXED; + if (argc == 3 && (strcmp("-S", argv[2]) == 0)) { + mode = SearchMode::STREAMING; + --argc; + } if (argc != 2) { return usage(); } - auto [ok, messages] = verifyRankSetup(argv[1]); + auto [ok, messages] = verifyRankSetup(argv[1], mode); for (const auto & msg : messages) { VLOG(toLogLevel(msg.first), "%s", msg.second.c_str()); -- cgit v1.2.3