diff options
Diffstat (limited to 'searchcore/src/tests')
13 files changed, 100 insertions, 5 deletions
diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 7ae828241c8..3f65f61cbaa 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -643,6 +643,7 @@ TEST("require that maintenance controller should change if some config has chang TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankProfilesChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankingConstantsChanged(true))); + TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankingExpressionsChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setIndexschemaChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setAttributesChanged(true))); @@ -690,6 +691,7 @@ TEST("require that subdbs should change if relevant config changed") TEST_DO(assertSubDbsShouldChange(CCR().setVisibilityDelayChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setRankProfilesChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true))); + TEST_DO(assertSubDbsShouldChange(CCR().setRankingExpressionsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().set_alloc_config_changed(true))); diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index aed01ca0192..544ff853384 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -17,6 +17,7 @@ using namespace search::index; using namespace search; using namespace vespa::config::search; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using std::make_shared; using std::shared_ptr; @@ -69,6 +70,11 @@ public: _builder.rankingConstants(make_shared<RankingConstants>(constants)); return *this; } + MyConfigBuilder &addRankingExpression() { + auto expr_list = RankingExpressions().add("my_expr", "my_file"); + _builder.rankingExpressions(make_shared<RankingExpressions>(expr_list)); + return *this; + } MyConfigBuilder &addOnnxModel() { OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; _builder.onnxModels(make_shared<OnnxModels>(models)); @@ -138,6 +144,7 @@ struct Fixture { fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). + addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(true). @@ -173,6 +180,7 @@ struct DelayAttributeAspectFixture { attrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(true)).addAttribute(). addRankProfile(). addRankingConstant(). + addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(true). @@ -180,6 +188,7 @@ struct DelayAttributeAspectFixture { build(); noAttrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(hasDocField)).addRankProfile(). addRankingConstant(). + addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(hasDocField). diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index e6bcbf18495..4ac66e464e2 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -28,6 +28,7 @@ using namespace vespa::config::search; using namespace std::chrono_literals; using vespa::config::content::core::BucketspacesConfig; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; typedef DocumentDBConfigHelper DBCM; @@ -76,8 +77,10 @@ assertEqualSnapshot(const DocumentDBConfig &exp, const DocumentDBConfig &act) { EXPECT_TRUE(exp.getRankProfilesConfig() == act.getRankProfilesConfig()); EXPECT_TRUE(exp.getRankingConstants() == act.getRankingConstants()); + EXPECT_TRUE(exp.getRankingExpressions() == act.getRankingExpressions()); EXPECT_TRUE(exp.getOnnxModels() == act.getOnnxModels()); EXPECT_EQUAL(0u, exp.getRankingConstants().size()); + EXPECT_EQUAL(0u, exp.getRankingExpressions().size()); EXPECT_EQUAL(0u, exp.getOnnxModels().size()); EXPECT_TRUE(exp.getIndexschemaConfig() == act.getIndexschemaConfig()); EXPECT_TRUE(exp.getAttributesConfig() == act.getAttributesConfig()); @@ -108,6 +111,9 @@ addConfigsThatAreNotSavedToDisk(const DocumentDBConfig &cfg) RankingConstants::Vector constants = {{"my_name", "my_type", "my_path"}}; builder.rankingConstants(std::make_shared<RankingConstants>(constants)); + auto expr_list = RankingExpressions().add("my_expr", "my_file"); + builder.rankingExpressions(std::make_shared<RankingExpressions>(expr_list)); + OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; builder.onnxModels(std::make_shared<OnnxModels>(models)); diff --git a/searchcore/src/tests/proton/matching/index_environment/.gitattributes b/searchcore/src/tests/proton/matching/index_environment/.gitattributes new file mode 100644 index 00000000000..c7aff0d0972 --- /dev/null +++ b/searchcore/src/tests/proton/matching/index_environment/.gitattributes @@ -0,0 +1 @@ +/my_expr.lz4 binary diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index 421ebffafa4..20b9e1206c7 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -3,6 +3,7 @@ #include <vespa/eval/eval/value_cache/constant_value.h> #include <vespa/searchcore/proton/matching/indexenvironment.h> +#include <vespa/searchcore/proton/matching/ranking_expressions.h> using namespace proton::matching; using search::fef::FieldInfo; @@ -15,6 +16,21 @@ using search::index::schema::DataType; using vespalib::eval::ConstantValue; using SIAF = Schema::ImportedAttributeField; +const vespalib::string my_expr_ref( + "this is my reference ranking expression.\n" + "this is my reference ranking expression.\n" + "it will not compile into a function.\n" + "it will not compile into a function.\n" + "it is just some text, that can also be compressed...\n" + "it is just some text, that can also be compressed...\n"); + +RankingExpressions make_expressions() { + RankingExpressions expr_list; + expr_list.add("expr1", TEST_PATH("my_expr")); + expr_list.add("expr2", TEST_PATH("my_expr.lz4")); + return expr_list; +} + OnnxModels make_models() { OnnxModels::Vector list; list.emplace_back(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1")); @@ -50,7 +66,7 @@ struct Fixture { Fixture(Schema::UP schema_) : repo(), schema(std::move(schema_)), - env(7, *schema, Properties(), repo, make_models()) + env(7, *schema, Properties(), repo, make_expressions(), make_models()) { } const FieldInfo *assertField(size_t idx, @@ -123,4 +139,13 @@ TEST_F("require that onnx model config can be obtained", Fixture(buildEmptySchem EXPECT_TRUE(f1.env.getOnnxModel("model3") == nullptr); } +TEST_F("require that external ranking expressions can be obtained", Fixture(buildEmptySchema())) { + auto expr1 = f1.env.getRankingExpression("expr1"); + auto expr2 = f1.env.getRankingExpression("expr2"); + auto expr3 = f1.env.getRankingExpression("expr3"); + EXPECT_EQUAL(expr1, my_expr_ref); + EXPECT_EQUAL(expr2, my_expr_ref); + EXPECT_TRUE(expr3.empty()); +} + TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/index_environment/my_expr b/searchcore/src/tests/proton/matching/index_environment/my_expr new file mode 100644 index 00000000000..2fdd1e4c9ff --- /dev/null +++ b/searchcore/src/tests/proton/matching/index_environment/my_expr @@ -0,0 +1,6 @@ +this is my reference ranking expression. +this is my reference ranking expression. +it will not compile into a function. +it will not compile into a function. +it is just some text, that can also be compressed... +it is just some text, that can also be compressed... diff --git a/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 b/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 Binary files differnew file mode 100644 index 00000000000..c88594ce81d --- /dev/null +++ b/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 8057dbf2da6..2bee53a5215 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -280,7 +280,7 @@ struct MyWorld { } Matcher::SP createMatcher() { - return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, OnnxModels(), 0); + return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), 0); } struct MySearchHandler : ISearchHandler { diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index e42f8323473..20a10aee678 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -10,6 +10,7 @@ #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/common/subdbtype.h> #include <vespa/searchcore/config/config-ranking-constants.h> +#include <vespa/searchcore/config/config-ranking-expressions.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/repo/documenttyperepo.h> @@ -52,6 +53,7 @@ struct DoctypeFixture { AttributesConfigBuilder attributesBuilder; RankProfilesConfigBuilder rankProfilesBuilder; RankingConstantsConfigBuilder rankingConstantsBuilder; + RankingExpressionsConfigBuilder rankingExpressionsBuilder; OnnxModelsConfigBuilder onnxModelsBuilder; IndexschemaConfigBuilder indexschemaBuilder; SummaryConfigBuilder summaryBuilder; @@ -108,6 +110,7 @@ struct ConfigTestFixture { set.addBuilder(db.configid, &fixture->attributesBuilder); set.addBuilder(db.configid, &fixture->rankProfilesBuilder); set.addBuilder(db.configid, &fixture->rankingConstantsBuilder); + set.addBuilder(db.configid, &fixture->rankingExpressionsBuilder); set.addBuilder(db.configid, &fixture->onnxModelsBuilder); set.addBuilder(db.configid, &fixture->indexschemaBuilder); set.addBuilder(db.configid, &fixture->summaryBuilder); @@ -262,7 +265,7 @@ TEST_FF("require that documentdb config manager subscribes for config", DocumentDBConfigManager(f1.configId + "/typea", "typea")) { f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); - ASSERT_EQUAL(9u, keySet.size()); + ASSERT_EQUAL(10u, keySet.size()); ASSERT_TRUE(f1.configEqual("typea", getDocumentDBConfig(f1, f2))); } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index 16e2fc4b1a8..faf09be3bbd 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -48,14 +48,13 @@ using std::map; using search::index::Schema; using search::index::SchemaBuilder; using proton::matching::RankingConstants; +using proton::matching::RankingExpressions; using proton::matching::OnnxModels; struct DBConfigFixture { using UP = std::unique_ptr<DBConfigFixture>; AttributesConfigBuilder _attributesBuilder; RankProfilesConfigBuilder _rankProfilesBuilder; - RankingConstantsConfigBuilder _rankingConstantsBuilder; - OnnxModelsConfigBuilder _onnxModelsBuilder; IndexschemaConfigBuilder _indexschemaBuilder; SummaryConfigBuilder _summaryBuilder; SummarymapConfigBuilder _summarymapBuilder; @@ -76,6 +75,11 @@ struct DBConfigFixture { return std::make_shared<RankingConstants>(); } + RankingExpressions::SP buildRankingExpressions() + { + return std::make_shared<RankingExpressions>(); + } + OnnxModels::SP buildOnnxModels() { return std::make_shared<OnnxModels>(); @@ -91,6 +95,7 @@ struct DBConfigFixture { (generation, std::make_shared<RankProfilesConfig>(_rankProfilesBuilder), buildRankingConstants(), + buildRankingExpressions(), buildOnnxModels(), std::make_shared<IndexschemaConfig>(_indexschemaBuilder), std::make_shared<AttributesConfig>(_attributesBuilder), diff --git a/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression b/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression new file mode 100644 index 00000000000..475b311dc84 --- /dev/null +++ b/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression @@ -0,0 +1 @@ +join(constant(my_tensor),attribute(date),f(t,d)(join(t,d,f(x,y)(x+y)))) diff --git a/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression b/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression new file mode 100644 index 00000000000..f6712be7a82 --- /dev/null +++ b/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression @@ -0,0 +1 @@ +join(constant(my_tensor),attribute(date),f(t,d)(t+d)) diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index c46990732b7..31557f13a54 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -71,6 +71,7 @@ struct Setup { std::map<std::string,std::string> properties; std::map<std::string,std::string> constants; std::vector<bool> extra_profiles; + std::map<std::string,std::string> ranking_expressions; std::map<std::string,OnnxModel> onnx_models; Setup(); ~Setup(); @@ -96,6 +97,9 @@ struct Setup { void rank_expr(const std::string &name, const std::string &expr) { property(fmt("rankingExpression(%s).rankingScript", name.c_str()), expr); } + void ext_rank_expr(const std::string &name, const std::string &file) { + ranking_expressions.insert_or_assign(name, TEST_PATH(file)); + } void first_phase(const std::string &feature) { property(rank::FirstPhase::NAME, feature); } @@ -157,6 +161,14 @@ struct Setup { ++idx; } } + void write_ranking_expressions(const Writer &out) { + size_t idx = 0; + for (const auto &entry: ranking_expressions) { + out.fmt("expression[%zu].name \"%s\"\n", idx, entry.first.c_str()); + out.fmt("expression[%zu].fileref \"expr_ref_%zu\"\n", idx, idx); + ++idx; + } + } void write_onnx_models(const Writer &out) { size_t idx = 0; for (const auto &entry: onnx_models) { @@ -179,6 +191,12 @@ struct Setup { } void write_self_cfg(const Writer &out) { size_t idx = 0; + for (const auto &entry: ranking_expressions) { + out.fmt("file[%zu].ref \"expr_ref_%zu\"\n", idx, idx); + out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.c_str()); + ++idx; + } + idx = 0; for (const auto &entry: onnx_models) { out.fmt("file[%zu].ref \"onnx_ref_%zu\"\n", idx, idx); out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.file_path().c_str()); @@ -190,6 +208,7 @@ struct Setup { write_indexschema(Writer(gen_dir + "/indexschema.cfg")); write_rank_profiles(Writer(gen_dir + "/rank-profiles.cfg")); write_ranking_constants(Writer(gen_dir + "/ranking-constants.cfg")); + write_ranking_expressions(Writer(gen_dir + "/ranking-expressions.cfg")); write_onnx_models(Writer(gen_dir + "/onnx-models.cfg")); write_self_cfg(Writer(gen_dir + "/verify-ranksetup.cfg")); } @@ -358,6 +377,23 @@ TEST_F("require that imported attribute field can be used by rank feature", Simp //----------------------------------------------------------------------------- +TEST_F("require that external ranking expression can be verified", SimpleSetup()) { + f.ext_rank_expr("my_expr", "good_ranking_expression"); + f.verify_valid({"rankingExpression(my_expr)"}); +} + +TEST_F("require that external ranking expression can fail verification", SimpleSetup()) { + f.ext_rank_expr("my_expr", "bad_ranking_expression"); + f.verify_invalid({"rankingExpression(my_expr)"}); +} + +TEST_F("require that missing expression file fails verification", SimpleSetup()) { + f.ext_rank_expr("my_expr", "missing_ranking_expression_file"); + f.verify_invalid({"rankingExpression(my_expr)"}); +} + +//----------------------------------------------------------------------------- + TEST_F("require that onnx model can be verified", OnnxSetup()) { f.rank_expr("query_tensor", "tensor<float>(a[1],b[4]):[[1,2,3,4]]"); f.rank_expr("attribute_tensor", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]"); |