From a6babd7ec4b48975fc5d6f2994b91bf3bfd2052c Mon Sep 17 00:00:00 2001 From: Håvard Pettersen Date: Wed, 12 May 2021 10:12:27 +0000 Subject: external ranking expressions loaded from potentially compressed files --- .../src/apps/tests/persistenceconformance_test.cpp | 1 + .../src/apps/verify_ranksetup/verify_ranksetup.cpp | 32 ++++++++-- .../src/apps/vespa-feed-bm/vespa_feed_bm.cpp | 1 + .../documentdb/configurer/configurer_test.cpp | 2 + .../documentdbconfig/documentdbconfig_test.cpp | 9 +++ .../fileconfigmanager/fileconfigmanager_test.cpp | 6 ++ .../matching/index_environment/.gitattributes | 1 + .../index_environment/index_environment_test.cpp | 27 ++++++++- .../proton/matching/index_environment/my_expr | 6 ++ .../proton/matching/index_environment/my_expr.lz4 | Bin 0 -> 164 bytes .../src/tests/proton/matching/matching_test.cpp | 2 +- .../proton_config_fetcher_test.cpp | 5 +- .../proton_configurer/proton_configurer_test.cpp | 9 ++- .../proton/verify_ranksetup/bad_ranking_expression | 1 + .../verify_ranksetup/good_ranking_expression | 1 + .../verify_ranksetup/verify_ranksetup_test.cpp | 36 +++++++++++ .../searchcore/proton/matching/CMakeLists.txt | 1 + .../proton/matching/indexenvironment.cpp | 24 +++++--- .../searchcore/proton/matching/indexenvironment.h | 5 ++ .../vespa/searchcore/proton/matching/matcher.cpp | 23 +++---- .../src/vespa/searchcore/proton/matching/matcher.h | 3 +- .../proton/matching/ranking_expressions.cpp | 67 +++++++++++++++++++++ .../proton/matching/ranking_expressions.h | 35 +++++++++++ .../searchcore/proton/server/documentdbconfig.cpp | 10 +++ .../searchcore/proton/server/documentdbconfig.h | 8 +++ .../proton/server/documentdbconfigmanager.cpp | 32 ++++++++++ .../vespa/searchcore/proton/server/matchers.cpp | 4 +- .../searchcore/proton/server/reconfig_params.cpp | 3 +- .../server/searchable_doc_subdb_configurer.cpp | 6 +- .../server/searchable_doc_subdb_configurer.h | 1 + .../proton/server/searchabledocsubdb.cpp | 3 +- .../proton/test/documentdb_config_builder.cpp | 3 + .../proton/test/documentdb_config_builder.h | 5 ++ 33 files changed, 339 insertions(+), 33 deletions(-) create mode 100644 searchcore/src/tests/proton/matching/index_environment/.gitattributes create mode 100644 searchcore/src/tests/proton/matching/index_environment/my_expr create mode 100644 searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 create mode 100644 searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression create mode 100644 searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression create mode 100644 searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp create mode 100644 searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h (limited to 'searchcore') diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 69d509c25fd..24db810b9df 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -133,6 +133,7 @@ public: 1, std::make_shared(), std::make_shared(), + std::make_shared(), std::make_shared(), indexschema, attributes, diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 559e8fd14cd..106b9bf4d24 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -15,8 +15,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -33,11 +35,13 @@ using config::ConfigSubscriber; using config::IConfigContext; using config::InvalidConfigException; using proton::matching::IConstantValueRepo; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using vespa::config::search::AttributesConfig; using vespa::config::search::IndexschemaConfig; using vespa::config::search::RankProfilesConfig; using vespa::config::search::core::RankingConstantsConfig; +using vespa::config::search::core::RankingExpressionsConfig; using vespa::config::search::core::OnnxModelsConfig; using vespa::config::search::core::VerifyRanksetupConfig; using vespalib::eval::BadConstantValue; @@ -57,6 +61,19 @@ std::optional get_file(const vespalib::string &ref, const Veri return std::nullopt; } +RankingExpressions make_expressions(const RankingExpressionsConfig &expressionsCfg, const VerifyRanksetupConfig &myCfg) { + RankingExpressions expressions; + for (const auto &entry: expressionsCfg.expression) { + if (auto file = get_file(entry.fileref, myCfg)) { + expressions.add(entry.name, file.value()); + } else { + LOG(warning, "could not find file for ranking expression '%s' (ref:'%s')\n", + entry.name.c_str(), entry.fileref.c_str()); + } + } + return expressions; +} + OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCfg) { OnnxModels::Vector model_list; for (const auto &entry: modelsCfg.model) { @@ -77,13 +94,15 @@ public: bool verify(const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &repo, - OnnxModels models); + const RankingExpressions &expressions, + const OnnxModels &models); bool verifyConfig(const VerifyRanksetupConfig &myCfg, const RankProfilesConfig &rankCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, const RankingConstantsConfig &constantsCfg, + const RankingExpressionsConfig &expressionsCfg, const OnnxModelsConfig &modelsCfg); int usage(); @@ -112,9 +131,10 @@ bool App::verify(const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &repo, - OnnxModels models) + const RankingExpressions &expressions, + const OnnxModels &models) { - proton::matching::IndexEnvironment indexEnv(0, schema, props, repo, models); + proton::matching::IndexEnvironment indexEnv(0, schema, props, repo, expressions, models); search::fef::BlueprintFactory factory; search::features::setup_search_features(factory); search::fef::test::setup_fef_test_plugin(factory); @@ -143,6 +163,7 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, const RankingConstantsConfig &constantsCfg, + const RankingExpressionsConfig &expressionsCfg, const OnnxModelsConfig &modelsCfg) { bool ok = true; @@ -150,6 +171,7 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, search::index::SchemaBuilder::build(schemaCfg, schema); search::index::SchemaBuilder::build(attributeCfg, schema); DummyConstantValueRepo repo(constantsCfg); + auto expressions = make_expressions(expressionsCfg, myCfg); auto models = make_models(modelsCfg, myCfg); for(size_t i = 0; i < rankCfg.rankprofile.size(); i++) { search::fef::Properties properties; @@ -158,7 +180,7 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, properties.add(profile.fef.property[j].name, profile.fef.property[j].value); } - if (verify(schema, properties, repo, models)) { + if (verify(schema, properties, repo, expressions, models)) { LOG(info, "rank profile '%s': pass", profile.name.c_str()); } else { LOG(error, "rank profile '%s': FAIL", profile.name.c_str()); @@ -196,6 +218,7 @@ App::Main() ConfigHandle::UP attributesHandle = subscriber.subscribe(cfgId); ConfigHandle::UP schemaHandle = subscriber.subscribe(cfgId); ConfigHandle::UP constantsHandle = subscriber.subscribe(cfgId); + ConfigHandle::UP expressionsHandle = subscriber.subscribe(cfgId); ConfigHandle::UP modelsHandle = subscriber.subscribe(cfgId); subscriber.nextConfig(); @@ -204,6 +227,7 @@ App::Main() *schemaHandle->getConfig(), *attributesHandle->getConfig(), *constantsHandle->getConfig(), + *expressionsHandle->getConfig(), *modelsHandle->getConfig()); } catch (ConfigRuntimeException & e) { LOG(error, "Unable to subscribe to config: %s", e.getMessage().c_str()); diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index 6e346bcfa60..e4dd4715ed1 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -187,6 +187,7 @@ std::shared_ptr make_document_db_config(std::shared_ptr(), std::make_shared(), + std::make_shared(), std::make_shared(), indexschema, attributes, diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 7ae828241c8..3f65f61cbaa 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -643,6 +643,7 @@ TEST("require that maintenance controller should change if some config has chang TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankProfilesChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankingConstantsChanged(true))); + TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankingExpressionsChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setIndexschemaChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setAttributesChanged(true))); @@ -690,6 +691,7 @@ TEST("require that subdbs should change if relevant config changed") TEST_DO(assertSubDbsShouldChange(CCR().setVisibilityDelayChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setRankProfilesChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setRankingExpressionsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().set_alloc_config_changed(true))); diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index aed01ca0192..544ff853384 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -17,6 +17,7 @@ using namespace search::index; using namespace search; using namespace vespa::config::search; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using std::make_shared; using std::shared_ptr; @@ -69,6 +70,11 @@ public: _builder.rankingConstants(make_shared(constants)); return *this; } + MyConfigBuilder &addRankingExpression() { + auto expr_list = RankingExpressions().add("my_expr", "my_file"); + _builder.rankingExpressions(make_shared(expr_list)); + return *this; + } MyConfigBuilder &addOnnxModel() { OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; _builder.onnxModels(make_shared(models)); @@ -138,6 +144,7 @@ struct Fixture { fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). + addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(true). @@ -173,6 +180,7 @@ struct DelayAttributeAspectFixture { attrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(true)).addAttribute(). addRankProfile(). addRankingConstant(). + addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(true). @@ -180,6 +188,7 @@ struct DelayAttributeAspectFixture { build(); noAttrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(hasDocField)).addRankProfile(). addRankingConstant(). + addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(hasDocField). diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index e6bcbf18495..4ac66e464e2 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -28,6 +28,7 @@ using namespace vespa::config::search; using namespace std::chrono_literals; using vespa::config::content::core::BucketspacesConfig; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; typedef DocumentDBConfigHelper DBCM; @@ -76,8 +77,10 @@ assertEqualSnapshot(const DocumentDBConfig &exp, const DocumentDBConfig &act) { EXPECT_TRUE(exp.getRankProfilesConfig() == act.getRankProfilesConfig()); EXPECT_TRUE(exp.getRankingConstants() == act.getRankingConstants()); + EXPECT_TRUE(exp.getRankingExpressions() == act.getRankingExpressions()); EXPECT_TRUE(exp.getOnnxModels() == act.getOnnxModels()); EXPECT_EQUAL(0u, exp.getRankingConstants().size()); + EXPECT_EQUAL(0u, exp.getRankingExpressions().size()); EXPECT_EQUAL(0u, exp.getOnnxModels().size()); EXPECT_TRUE(exp.getIndexschemaConfig() == act.getIndexschemaConfig()); EXPECT_TRUE(exp.getAttributesConfig() == act.getAttributesConfig()); @@ -108,6 +111,9 @@ addConfigsThatAreNotSavedToDisk(const DocumentDBConfig &cfg) RankingConstants::Vector constants = {{"my_name", "my_type", "my_path"}}; builder.rankingConstants(std::make_shared(constants)); + auto expr_list = RankingExpressions().add("my_expr", "my_file"); + builder.rankingExpressions(std::make_shared(expr_list)); + OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; builder.onnxModels(std::make_shared(models)); diff --git a/searchcore/src/tests/proton/matching/index_environment/.gitattributes b/searchcore/src/tests/proton/matching/index_environment/.gitattributes new file mode 100644 index 00000000000..c7aff0d0972 --- /dev/null +++ b/searchcore/src/tests/proton/matching/index_environment/.gitattributes @@ -0,0 +1 @@ +/my_expr.lz4 binary diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index 421ebffafa4..20b9e1206c7 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -3,6 +3,7 @@ #include #include +#include using namespace proton::matching; using search::fef::FieldInfo; @@ -15,6 +16,21 @@ using search::index::schema::DataType; using vespalib::eval::ConstantValue; using SIAF = Schema::ImportedAttributeField; +const vespalib::string my_expr_ref( + "this is my reference ranking expression.\n" + "this is my reference ranking expression.\n" + "it will not compile into a function.\n" + "it will not compile into a function.\n" + "it is just some text, that can also be compressed...\n" + "it is just some text, that can also be compressed...\n"); + +RankingExpressions make_expressions() { + RankingExpressions expr_list; + expr_list.add("expr1", TEST_PATH("my_expr")); + expr_list.add("expr2", TEST_PATH("my_expr.lz4")); + return expr_list; +} + OnnxModels make_models() { OnnxModels::Vector list; list.emplace_back(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1")); @@ -50,7 +66,7 @@ struct Fixture { Fixture(Schema::UP schema_) : repo(), schema(std::move(schema_)), - env(7, *schema, Properties(), repo, make_models()) + env(7, *schema, Properties(), repo, make_expressions(), make_models()) { } const FieldInfo *assertField(size_t idx, @@ -123,4 +139,13 @@ TEST_F("require that onnx model config can be obtained", Fixture(buildEmptySchem EXPECT_TRUE(f1.env.getOnnxModel("model3") == nullptr); } +TEST_F("require that external ranking expressions can be obtained", Fixture(buildEmptySchema())) { + auto expr1 = f1.env.getRankingExpression("expr1"); + auto expr2 = f1.env.getRankingExpression("expr2"); + auto expr3 = f1.env.getRankingExpression("expr3"); + EXPECT_EQUAL(expr1, my_expr_ref); + EXPECT_EQUAL(expr2, my_expr_ref); + EXPECT_TRUE(expr3.empty()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/index_environment/my_expr b/searchcore/src/tests/proton/matching/index_environment/my_expr new file mode 100644 index 00000000000..2fdd1e4c9ff --- /dev/null +++ b/searchcore/src/tests/proton/matching/index_environment/my_expr @@ -0,0 +1,6 @@ +this is my reference ranking expression. +this is my reference ranking expression. +it will not compile into a function. +it will not compile into a function. +it is just some text, that can also be compressed... +it is just some text, that can also be compressed... diff --git a/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 b/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 new file mode 100644 index 00000000000..c88594ce81d Binary files /dev/null and b/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 differ diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 8057dbf2da6..2bee53a5215 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -280,7 +280,7 @@ struct MyWorld { } Matcher::SP createMatcher() { - return std::make_shared(schema, config, clock, queryLimiter, constantValueRepo, OnnxModels(), 0); + return std::make_shared(schema, config, clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), 0); } struct MySearchHandler : ISearchHandler { diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index e42f8323473..20a10aee678 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -52,6 +53,7 @@ struct DoctypeFixture { AttributesConfigBuilder attributesBuilder; RankProfilesConfigBuilder rankProfilesBuilder; RankingConstantsConfigBuilder rankingConstantsBuilder; + RankingExpressionsConfigBuilder rankingExpressionsBuilder; OnnxModelsConfigBuilder onnxModelsBuilder; IndexschemaConfigBuilder indexschemaBuilder; SummaryConfigBuilder summaryBuilder; @@ -108,6 +110,7 @@ struct ConfigTestFixture { set.addBuilder(db.configid, &fixture->attributesBuilder); set.addBuilder(db.configid, &fixture->rankProfilesBuilder); set.addBuilder(db.configid, &fixture->rankingConstantsBuilder); + set.addBuilder(db.configid, &fixture->rankingExpressionsBuilder); set.addBuilder(db.configid, &fixture->onnxModelsBuilder); set.addBuilder(db.configid, &fixture->indexschemaBuilder); set.addBuilder(db.configid, &fixture->summaryBuilder); @@ -262,7 +265,7 @@ TEST_FF("require that documentdb config manager subscribes for config", DocumentDBConfigManager(f1.configId + "/typea", "typea")) { f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); - ASSERT_EQUAL(9u, keySet.size()); + ASSERT_EQUAL(10u, keySet.size()); ASSERT_TRUE(f1.configEqual("typea", getDocumentDBConfig(f1, f2))); } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 16e2fc4b1a8..faf09be3bbd 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -48,14 +48,13 @@ using std::map; using search::index::Schema; using search::index::SchemaBuilder; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; struct DBConfigFixture { using UP = std::unique_ptr; AttributesConfigBuilder _attributesBuilder; RankProfilesConfigBuilder _rankProfilesBuilder; - RankingConstantsConfigBuilder _rankingConstantsBuilder; - OnnxModelsConfigBuilder _onnxModelsBuilder; IndexschemaConfigBuilder _indexschemaBuilder; SummaryConfigBuilder _summaryBuilder; SummarymapConfigBuilder _summarymapBuilder; @@ -76,6 +75,11 @@ struct DBConfigFixture { return std::make_shared(); } + RankingExpressions::SP buildRankingExpressions() + { + return std::make_shared(); + } + OnnxModels::SP buildOnnxModels() { return std::make_shared(); @@ -91,6 +95,7 @@ struct DBConfigFixture { (generation, std::make_shared(_rankProfilesBuilder), buildRankingConstants(), + buildRankingExpressions(), buildOnnxModels(), std::make_shared(_indexschemaBuilder), std::make_shared(_attributesBuilder), diff --git a/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression b/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression new file mode 100644 index 00000000000..475b311dc84 --- /dev/null +++ b/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression @@ -0,0 +1 @@ +join(constant(my_tensor),attribute(date),f(t,d)(join(t,d,f(x,y)(x+y)))) diff --git a/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression b/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression new file mode 100644 index 00000000000..f6712be7a82 --- /dev/null +++ b/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression @@ -0,0 +1 @@ +join(constant(my_tensor),attribute(date),f(t,d)(t+d)) diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index c46990732b7..31557f13a54 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -71,6 +71,7 @@ struct Setup { std::map properties; std::map constants; std::vector extra_profiles; + std::map ranking_expressions; std::map onnx_models; Setup(); ~Setup(); @@ -96,6 +97,9 @@ struct Setup { void rank_expr(const std::string &name, const std::string &expr) { property(fmt("rankingExpression(%s).rankingScript", name.c_str()), expr); } + void ext_rank_expr(const std::string &name, const std::string &file) { + ranking_expressions.insert_or_assign(name, TEST_PATH(file)); + } void first_phase(const std::string &feature) { property(rank::FirstPhase::NAME, feature); } @@ -157,6 +161,14 @@ struct Setup { ++idx; } } + void write_ranking_expressions(const Writer &out) { + size_t idx = 0; + for (const auto &entry: ranking_expressions) { + out.fmt("expression[%zu].name \"%s\"\n", idx, entry.first.c_str()); + out.fmt("expression[%zu].fileref \"expr_ref_%zu\"\n", idx, idx); + ++idx; + } + } void write_onnx_models(const Writer &out) { size_t idx = 0; for (const auto &entry: onnx_models) { @@ -179,6 +191,12 @@ struct Setup { } void write_self_cfg(const Writer &out) { size_t idx = 0; + for (const auto &entry: ranking_expressions) { + out.fmt("file[%zu].ref \"expr_ref_%zu\"\n", idx, idx); + out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.c_str()); + ++idx; + } + idx = 0; for (const auto &entry: onnx_models) { out.fmt("file[%zu].ref \"onnx_ref_%zu\"\n", idx, idx); out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.file_path().c_str()); @@ -190,6 +208,7 @@ struct Setup { write_indexschema(Writer(gen_dir + "/indexschema.cfg")); write_rank_profiles(Writer(gen_dir + "/rank-profiles.cfg")); write_ranking_constants(Writer(gen_dir + "/ranking-constants.cfg")); + write_ranking_expressions(Writer(gen_dir + "/ranking-expressions.cfg")); write_onnx_models(Writer(gen_dir + "/onnx-models.cfg")); write_self_cfg(Writer(gen_dir + "/verify-ranksetup.cfg")); } @@ -358,6 +377,23 @@ TEST_F("require that imported attribute field can be used by rank feature", Simp //----------------------------------------------------------------------------- +TEST_F("require that external ranking expression can be verified", SimpleSetup()) { + f.ext_rank_expr("my_expr", "good_ranking_expression"); + f.verify_valid({"rankingExpression(my_expr)"}); +} + +TEST_F("require that external ranking expression can fail verification", SimpleSetup()) { + f.ext_rank_expr("my_expr", "bad_ranking_expression"); + f.verify_invalid({"rankingExpression(my_expr)"}); +} + +TEST_F("require that missing expression file fails verification", SimpleSetup()) { + f.ext_rank_expr("my_expr", "missing_ranking_expression_file"); + f.verify_invalid({"rankingExpression(my_expr)"}); +} + +//----------------------------------------------------------------------------- + TEST_F("require that onnx model can be verified", OnnxSetup()) { f.rank_expr("query_tensor", "tensor(a[1],b[4]):[[1,2,3,4]]"); f.rank_expr("attribute_tensor", "tensor(a[4],b[1]):[[5],[6],[7],[8]]"); diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index a4688b5fdca..d83d8ed91d9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -27,6 +27,7 @@ vespa_add_library(searchcore_matching STATIC querylimiter.cpp querynodes.cpp ranking_constants.cpp + ranking_expressions.cpp requestcontext.cpp resolveviewvisitor.cpp result_processor.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp index 013f359c4f9..5f5cc4dcf76 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp @@ -65,15 +65,17 @@ IndexEnvironment::IndexEnvironment(uint32_t distributionKey, const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, OnnxModels onnxModels) - : _tableManager(), - _properties(props), - _fieldNames(), - _fields(), - _motivation(UNKNOWN), - _constantValueRepo(constantValueRepo), - _onnxModels(std::move(onnxModels)), - _distributionKey(distributionKey) + : _tableManager(), + _properties(props), + _fieldNames(), + _fields(), + _motivation(UNKNOWN), + _constantValueRepo(constantValueRepo), + _rankingExpressions(std::move(rankingExpressions)), + _onnxModels(std::move(onnxModels)), + _distributionKey(distributionKey) { _tableManager.addFactory(std::make_shared(256)); extractFields(schema); @@ -131,6 +133,12 @@ IndexEnvironment::hintFieldAccess(uint32_t ) const { } void IndexEnvironment::hintAttributeAccess(const string &) const { } +vespalib::string +IndexEnvironment::getRankingExpression(const vespalib::string &name) const +{ + return _rankingExpressions.loadExpression(name); +} + const search::fef::OnnxModel * IndexEnvironment::getOnnxModel(const vespalib::string &name) const { diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h index ad51eb17b4d..d270825d3f7 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h @@ -3,6 +3,7 @@ #pragma once #include "onnx_models.h" +#include "ranking_expressions.h" #include "i_constant_value_repo.h" #include #include @@ -26,6 +27,7 @@ private: std::vector _fields; mutable FeatureMotivation _motivation; const IConstantValueRepo &_constantValueRepo; + RankingExpressions _rankingExpressions; OnnxModels _onnxModels; uint32_t _distributionKey; @@ -46,12 +48,14 @@ public: * @param schema the index schema * @param props config * @param constantValueRepo repo used to access constant values for ranking + * @param rankingExpressions processed config about ranking expressions * @param onnxModels processed config about onnx models **/ IndexEnvironment(uint32_t distributionKey, const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, OnnxModels onnxModels); const search::fef::Properties &getProperties() const override; @@ -68,6 +72,7 @@ public: vespalib::eval::ConstantValue::UP getConstantValue(const vespalib::string &name) const override { return _constantValueRepo.getConstant(name); } + vespalib::string getRankingExpression(const vespalib::string &name) const override; const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override; ~IndexEnvironment() override; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index e459a45040b..806753a6660 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -99,17 +99,18 @@ handleGroupingSession(SessionManager &sessionMgr, GroupingContext & groupingCont } // namespace proton::matching:: Matcher::Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, - QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, OnnxModels onnxModels, uint32_t distributionKey) - : _indexEnv(distributionKey, schema, props, constantValueRepo, std::move(onnxModels)), - _blueprintFactory(), - _rankSetup(), - _viewResolver(ViewResolver::createFromSchema(schema)), - _statsLock(), - _stats(), - _startTime(my_clock::now()), - _clock(clock), - _queryLimiter(queryLimiter), - _distributionKey(distributionKey) + QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, OnnxModels onnxModels, uint32_t distributionKey) + : _indexEnv(distributionKey, schema, props, constantValueRepo, std::move(rankingExpressions), std::move(onnxModels)), + _blueprintFactory(), + _rankSetup(), + _viewResolver(ViewResolver::createFromSchema(schema)), + _statsLock(), + _stats(), + _startTime(my_clock::now()), + _clock(clock), + _queryLimiter(queryLimiter), + _distributionKey(distributionKey) { search::features::setup_search_features(_blueprintFactory); search::fef::test::setup_fef_test_plugin(_blueprintFactory); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index dcd1bbb2b46..d40ae9b18f1 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -88,7 +88,8 @@ public: **/ Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, QueryLimiter &queryLimiter, - const IConstantValueRepo &constantValueRepo, OnnxModels onnxModels, + const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, OnnxModels onnxModels, uint32_t distributionKey); const search::fef::IIndexEnvironment &get_index_env() const { return _indexEnv; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp new file mode 100644 index 00000000000..9b74f76aa6e --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp @@ -0,0 +1,67 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "ranking_expressions.h" +#include +#include +#include + +#include +LOG_SETUP(".proton.matching.ranking_expressions"); + +namespace proton::matching { + +namespace { + +vespalib::string extract_data(vespalib::Input &input) { + vespalib::string result; + for (auto chunk = input.obtain(); chunk.size > 0; chunk = input.obtain()) { + result.append(vespalib::stringref(chunk.data, chunk.size)); + input.evict(chunk.size); + } + return result; +} + +} // unnamed + +RankingExpressions::RankingExpressions() = default; +RankingExpressions::RankingExpressions(RankingExpressions &&rhs) = default; +RankingExpressions::RankingExpressions(const RankingExpressions &rhs) = default; +RankingExpressions::~RankingExpressions() = default; + +RankingExpressions & +RankingExpressions::add(const vespalib::string &name, const vespalib::string &path) +{ + _expressions.insert_or_assign(name, path); + return *this; +} + +vespalib::string +RankingExpressions::loadExpression(const vespalib::string &name) const +{ + auto pos = _expressions.find(name); + if (pos == _expressions.end()) { + // not warning about missing expression here since what we + // think is a name might be an expression itself. + return {}; + } + auto path = pos->second; + vespalib::MappedFileInput file(path); + if (!file.valid()) { + LOG(warning, "could not read file: %s", path.c_str()); + return {}; + } + if (ends_with(path, ".lz4")) { + size_t buffer_size = 64_Ki; + vespalib::Lz4InputDecoder lz4_decoder(file, buffer_size); + auto result = extract_data(lz4_decoder); + if (lz4_decoder.failed()) { + LOG(warning, "file contains lz4 errors (%s): %s", + lz4_decoder.reason().c_str(), path.c_str()); + return {}; + } + return result; + } + return extract_data(file); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h new file mode 100644 index 00000000000..e1472b03ee0 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h @@ -0,0 +1,35 @@ +// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include +#include +#include + +namespace proton::matching { + +/** + * Class representing a collection of named ranking expressions + * obtained through file-distribution. + */ +class RankingExpressions +{ +private: + // expression name -> full_path of expression file + std::map _expressions; + +public: + RankingExpressions(); + RankingExpressions(RankingExpressions &&rhs); + RankingExpressions(const RankingExpressions &rhs); + ~RankingExpressions(); + using SP = std::shared_ptr; + bool operator==(const RankingExpressions &rhs) const { + return _expressions == rhs._expressions; + } + size_t size() const { return _expressions.size(); } + RankingExpressions &add(const vespalib::string &name, const vespalib::string &path); + vespalib::string loadExpression(const vespalib::string &name) const; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index 7c487043b5b..e39188042d7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -35,6 +35,7 @@ namespace proton { DocumentDBConfig::ComparisonResult::ComparisonResult() : rankProfilesChanged(false), rankingConstantsChanged(false), + rankingExpressionsChanged(false), onnxModelsChanged(false), indexschemaChanged(false), attributesChanged(false), @@ -58,6 +59,7 @@ DocumentDBConfig::DocumentDBConfig( int64_t generation, const RankProfilesConfigSP &rankProfiles, const RankingConstants::SP &rankingConstants, + const RankingExpressions::SP &rankingExpressions, const OnnxModels::SP &onnxModels, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, @@ -80,6 +82,7 @@ DocumentDBConfig::DocumentDBConfig( _generation(generation), _rankProfiles(rankProfiles), _rankingConstants(rankingConstants), + _rankingExpressions(rankingExpressions), _onnxModels(onnxModels), _indexschema(indexschema), _attributes(attributes), @@ -107,6 +110,7 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _generation(cfg._generation), _rankProfiles(cfg._rankProfiles), _rankingConstants(cfg._rankingConstants), + _rankingExpressions(cfg._rankingExpressions), _onnxModels(cfg._onnxModels), _indexschema(cfg._indexschema), _attributes(cfg._attributes), @@ -133,6 +137,7 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const { return equals(_rankProfiles.get(), rhs._rankProfiles.get()) && equals(_rankingConstants.get(), rhs._rankingConstants.get()) && + equals(_rankingExpressions.get(), rhs._rankingExpressions.get()) && equals(_onnxModels.get(), rhs._onnxModels.get()) && equals(_indexschema.get(), rhs._indexschema.get()) && equals(_attributes.get(), rhs._attributes.get()) && @@ -157,6 +162,7 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const ComparisonResult retval; retval.rankProfilesChanged = !equals(_rankProfiles.get(), rhs._rankProfiles.get()); retval.rankingConstantsChanged = !equals(_rankingConstants.get(), rhs._rankingConstants.get()); + retval.rankingExpressionsChanged = !equals(_rankingExpressions.get(), rhs._rankingExpressions.get()); retval.onnxModelsChanged = !equals(_onnxModels.get(), rhs._onnxModels.get()); retval.indexschemaChanged = !equals(_indexschema.get(), rhs._indexschema.get()); retval.attributesChanged = !equals(_attributes.get(), rhs._attributes.get()); @@ -183,6 +189,7 @@ DocumentDBConfig::valid() const { return _rankProfiles && _rankingConstants && + _rankingExpressions && _onnxModels && _indexschema && _attributes && @@ -226,6 +233,7 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) o._generation, emptyConfig(o._rankProfiles), std::make_shared(), + std::make_shared(), std::make_shared(), o._indexschema, o._attributes, @@ -269,6 +277,7 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) _generation, _rankProfiles, _rankingConstants, + _rankingExpressions, _onnxModels, _indexschema, attributes, @@ -307,6 +316,7 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum (n._generation, n._rankProfiles, n._rankingConstants, + n._rankingExpressions, n._onnxModels, n._indexschema, attributeAspectDelayer.getAttributesConfig(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 8e24ed8e96a..2ca7fb1dfa8 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +41,7 @@ public: public: bool rankProfilesChanged; bool rankingConstantsChanged; + bool rankingExpressionsChanged; bool onnxModelsChanged; bool indexschemaChanged; bool attributesChanged; @@ -61,6 +63,7 @@ public: ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } ComparisonResult &setRankingConstantsChanged(bool val) { rankingConstantsChanged = val; return *this; } + ComparisonResult &setRankingExpressionsChanged(bool val) { rankingExpressionsChanged = val; return *this; } ComparisonResult &setOnnxModelsChanged(bool val) { onnxModelsChanged = val; return *this; } ComparisonResult &setIndexschemaChanged(bool val) { indexschemaChanged = val; return *this; } ComparisonResult &setAttributesChanged(bool val) { attributesChanged = val; return *this; } @@ -101,6 +104,7 @@ public: using RankProfilesConfig = const vespa::config::search::internal::InternalRankProfilesType; using RankProfilesConfigSP = std::shared_ptr; using RankingConstants = matching::RankingConstants; + using RankingExpressions = matching::RankingExpressions; using OnnxModels = matching::OnnxModels; using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; using SummaryConfigSP = std::shared_ptr; @@ -120,6 +124,7 @@ private: int64_t _generation; RankProfilesConfigSP _rankProfiles; RankingConstants::SP _rankingConstants; + RankingExpressions::SP _rankingExpressions; OnnxModels::SP _onnxModels; IndexschemaConfigSP _indexschema; AttributesConfigSP _attributes; @@ -159,6 +164,7 @@ public: DocumentDBConfig(int64_t generation, const RankProfilesConfigSP &rankProfiles, const RankingConstants::SP &rankingConstants, + const RankingExpressions::SP &rankingExpressions, const OnnxModels::SP &onnxModels, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, @@ -189,6 +195,7 @@ public: const RankProfilesConfig &getRankProfilesConfig() const { return *_rankProfiles; } const RankingConstants &getRankingConstants() const { return *_rankingConstants; } + const RankingExpressions &getRankingExpressions() const { return *_rankingExpressions; } const OnnxModels &getOnnxModels() const { return *_onnxModels; } const IndexschemaConfig &getIndexschemaConfig() const { return *_indexschema; } const AttributesConfig &getAttributesConfig() const { return *_attributes; } @@ -198,6 +205,7 @@ public: const DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } const RankProfilesConfigSP &getRankProfilesConfigSP() const { return _rankProfiles; } const RankingConstants::SP &getRankingConstantsSP() const { return _rankingConstants; } + const RankingExpressions::SP &getRankingExpressionsSP() const { return _rankingExpressions; } const OnnxModels::SP &getOnnxModelsSP() const { return _onnxModels; } const IndexschemaConfigSP &getIndexschemaConfigSP() const { return _indexschema; } const AttributesConfigSP &getAttributesConfigSP() const { return _attributes; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 8d513fde62f..c61e67facbf 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,7 @@ using search::TuneFileDocumentDB; using search::index::Schema; using search::index::SchemaBuilder; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using vespalib::compression::CompressionConfig; using search::LogDocumentStore; @@ -53,6 +55,7 @@ DocumentDBConfigManager::createConfigKeySet() const ConfigKeySet set; set.add; + using RankingExpressionsConfigSP = std::shared_ptr; using OnnxModelsConfigSP = std::shared_ptr; using IndexschemaConfigSP = DocumentDBConfig::IndexschemaConfigSP; using SummaryConfigSP = DocumentDBConfig::SummaryConfigSP; @@ -282,6 +286,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) DocumentDBConfig::SP current = _pendingConfigSnapshot; RankProfilesConfigSP newRankProfilesConfig; matching::RankingConstants::SP newRankingConstants; + matching::RankingExpressions::SP newRankingExpressions; matching::OnnxModels::SP newOnnxModels; IndexschemaConfigSP newIndexschemaConfig; MaintenanceConfigSP oldMaintenanceConfig; @@ -308,6 +313,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) if (current) { newRankProfilesConfig = current->getRankProfilesConfigSP(); newRankingConstants = current->getRankingConstantsSP(); + newRankingExpressions = current->getRankingExpressionsSP(); newOnnxModels = current->getOnnxModelsSP(); newIndexschemaConfig = current->getIndexschemaConfigSP(); oldMaintenanceConfig = current->getMaintenanceConfigSP(); @@ -344,6 +350,31 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) } newRankingConstants = std::make_shared(constants); } + if (snapshot.isChanged(_configId, currentGeneration)) { + RankingExpressionsConfigSP newRankingExpressionsConfig = RankingExpressionsConfigSP( + snapshot.getConfig(_configId)); + const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec; + RankingExpressions expressions; + if (spec != "") { + config::RpcFileAcquirer fileAcquirer(spec); + vespalib::TimeBox timeBox(5*60, 5); + for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) { + vespalib::string filePath; + LOG(info, "Waiting for file acquirer (name='%s', ref='%s')", + rc.name.c_str(), rc.fileref.c_str()); + while (timeBox.hasTimeLeft() && (filePath == "")) { + filePath = fileAcquirer.wait_for(rc.fileref, timeBox.timeLeft()); + if (filePath == "") { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + LOG(info, "Got file path from file acquirer: '%s' (name='%s', ref='%s')", + filePath.c_str(), rc.name.c_str(), rc.fileref.c_str()); + expressions.add(rc.name, filePath); + } + } + newRankingExpressions = std::make_shared(std::move(expressions)); + } if (snapshot.isChanged(_configId, currentGeneration)) { OnnxModelsConfigSP newOnnxModelsConfig = OnnxModelsConfigSP( snapshot.getConfig(_configId)); @@ -403,6 +434,7 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) auto newSnapshot = std::make_shared(generation, newRankProfilesConfig, newRankingConstants, + newRankingExpressions, newOnnxModels, newIndexschemaConfig, filterImportedAttributes(newAttributesConfig), diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp index 53c96a81134..7288e3e98fb 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp @@ -2,11 +2,13 @@ #include "matchers.h" #include +#include #include #include namespace proton { +using matching::RankingExpressions; using matching::OnnxModels; Matchers::Matchers(const vespalib::Clock &clock, @@ -14,7 +16,7 @@ Matchers::Matchers(const vespalib::Clock &clock, const matching::IConstantValueRepo &constantValueRepo) : _rpmap(), _fallback(new matching::Matcher(search::index::Schema(), search::fef::Properties(), - clock, queryLimiter, constantValueRepo, OnnxModels(), -1)), + clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), -1)), _default() { } diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index bb6c5423175..7cdc523e381 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -15,6 +15,7 @@ ReconfigParams::configHasChanged() const { return _res.rankProfilesChanged || _res.rankingConstantsChanged || + _res.rankingExpressionsChanged || _res.onnxModelsChanged || _res.indexschemaChanged || _res.attributesChanged || @@ -40,7 +41,7 @@ ReconfigParams::shouldSchemaChange() const bool ReconfigParams::shouldMatchersChange() const { - return _res.rankProfilesChanged || _res.rankingConstantsChanged || _res.onnxModelsChanged || shouldSchemaChange(); + return _res.rankProfilesChanged || _res.rankingConstantsChanged || _res.rankingExpressionsChanged || _res.onnxModelsChanged || shouldSchemaChange(); } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 1e5c558af0b..05e68ee0ec4 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -21,6 +21,7 @@ using vespa::config::search::RankProfilesConfig; namespace proton { using matching::Matcher; +using matching::RankingExpressions; using matching::OnnxModels; typedef AttributeReprocessingInitializer::Config ARIConfig; @@ -107,6 +108,7 @@ SearchableDocSubDBConfigurer::~SearchableDocSubDBConfigurer() = default; Matchers::UP SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, const RankProfilesConfig &cfg, + const RankingExpressions &rankingExpressions, const OnnxModels &onnxModels) { auto newMatchers = std::make_unique(_clock, _queryLimiter, _constantValueRepo); @@ -117,7 +119,8 @@ SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, properties.add(property.name, property.value); } // schema instance only used during call. - auto profptr = std::make_shared(*schema, properties, _clock, _queryLimiter, _constantValueRepo, onnxModels, _distributionKey); + auto profptr = std::make_shared(*schema, properties, _clock, _queryLimiter, _constantValueRepo, + rankingExpressions, onnxModels, _distributionKey); newMatchers->add(name, profptr); } return newMatchers; @@ -185,6 +188,7 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, _constantValueRepo.reconfigure(newConfig.getRankingConstants()); Matchers::SP newMatchers = createMatchers(newConfig.getSchemaSP(), newConfig.getRankProfilesConfig(), + newConfig.getRankingExpressions(), newConfig.getOnnxModels()); matchers = newMatchers; shouldMatchViewChange = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index d460bb6506f..1a9ba1abff7 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -74,6 +74,7 @@ public: Matchers::UP createMatchers(const search::index::Schema::SP &schema, const vespa::config::search::RankProfilesConfig &cfg, + const proton::matching::RankingExpressions &rankingExpressions, const proton::matching::OnnxModels &onnxModels); void reconfigureIndexSearchable(); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index 51e6f8e45df..d7aaa901530 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -201,7 +201,8 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess const Schema::SP &schema = configSnapshot.getSchemaSP(); const IIndexManager::SP &indexMgr = getIndexManager(); _constantValueRepo.reconfigure(configSnapshot.getRankingConstants()); - Matchers::SP matchers = _configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig(), configSnapshot.getOnnxModels()); + Matchers::SP matchers = _configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig(), + configSnapshot.getRankingExpressions(), configSnapshot.getOnnxModels()); auto matchView = std::make_shared(std::move(matchers), indexMgr->getSearchable(), attrMgr, sessionManager, _metaStoreCtx, _docIdLimit); _rSearchView.set(SearchView::create( diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 17cb91ac4ce..18774e60202 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -34,6 +34,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, : _generation(generation), _rankProfiles(std::make_shared()), _rankingConstants(std::make_shared()), + _rankingExpressions(std::make_shared()), _onnxModels(std::make_shared()), _indexschema(std::make_shared()), _attributes(std::make_shared()), @@ -58,6 +59,7 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) : _generation(cfg.getGeneration()), _rankProfiles(cfg.getRankProfilesConfigSP()), _rankingConstants(cfg.getRankingConstantsSP()), + _rankingExpressions(cfg.getRankingExpressionsSP()), _onnxModels(cfg.getOnnxModelsSP()), _indexschema(cfg.getIndexschemaConfigSP()), _attributes(cfg.getAttributesConfigSP()), @@ -86,6 +88,7 @@ DocumentDBConfigBuilder::build() _generation, _rankProfiles, _rankingConstants, + _rankingExpressions, _onnxModels, _indexschema, _attributes, diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index 706e14e73db..a248ed5544a 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -14,6 +14,7 @@ private: int64_t _generation; DocumentDBConfig::RankProfilesConfigSP _rankProfiles; DocumentDBConfig::RankingConstants::SP _rankingConstants; + DocumentDBConfig::RankingExpressions::SP _rankingExpressions; DocumentDBConfig::OnnxModels::SP _onnxModels; DocumentDBConfig::IndexschemaConfigSP _indexschema; DocumentDBConfig::AttributesConfigSP _attributes; @@ -57,6 +58,10 @@ public: _rankingConstants = rankingConstants_in; return *this; } + DocumentDBConfigBuilder &rankingExpressions(const DocumentDBConfig::RankingExpressions::SP &rankingExpressions_in) { + _rankingExpressions = rankingExpressions_in; + return *this; + } DocumentDBConfigBuilder &onnxModels(const DocumentDBConfig::OnnxModels::SP &onnxModels_in) { _onnxModels = onnxModels_in; return *this; -- cgit v1.2.3