diff options
39 files changed, 63 insertions, 410 deletions
diff --git a/searchcore/src/apps/tests/persistenceconformance_test.cpp b/searchcore/src/apps/tests/persistenceconformance_test.cpp index 24db810b9df..69d509c25fd 100644 --- a/searchcore/src/apps/tests/persistenceconformance_test.cpp +++ b/searchcore/src/apps/tests/persistenceconformance_test.cpp @@ -133,7 +133,6 @@ public: 1, std::make_shared<RankProfilesConfig>(), std::make_shared<matching::RankingConstants>(), - std::make_shared<matching::RankingExpressions>(), std::make_shared<matching::OnnxModels>(), indexschema, attributes, diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 853395be3e1..559e8fd14cd 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -15,10 +15,8 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/searchcommon/common/schemaconfigurer.h> #include <vespa/searchcore/config/config-ranking-constants.h> -#include <vespa/searchcore/config/config-ranking-expressions.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/searchcore/proton/matching/indexenvironment.h> -#include <vespa/searchcore/proton/matching/ranking_expressions.h> #include <vespa/searchlib/features/setup.h> #include <vespa/searchlib/fef/fef.h> #include <vespa/searchlib/fef/test/plugin/setup.h> @@ -35,13 +33,11 @@ using config::ConfigSubscriber; using config::IConfigContext; using config::InvalidConfigException; using proton::matching::IConstantValueRepo; -using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using vespa::config::search::AttributesConfig; using vespa::config::search::IndexschemaConfig; using vespa::config::search::RankProfilesConfig; using vespa::config::search::core::RankingConstantsConfig; -using vespa::config::search::core::RankingExpressionsConfig; using vespa::config::search::core::OnnxModelsConfig; using vespa::config::search::core::VerifyRanksetupConfig; using vespalib::eval::BadConstantValue; @@ -61,19 +57,6 @@ std::optional<vespalib::string> get_file(const vespalib::string &ref, const Veri return std::nullopt; } -RankingExpressions make_expressions(const RankingExpressionsConfig &expressionsCfg, const VerifyRanksetupConfig &myCfg) { - RankingExpressions expressions; - for (const auto &entry: expressionsCfg.expression) { - if (auto file = get_file(entry.fileref, myCfg)) { - expressions.add(entry.name, file.value()); - } else { - LOG(warning, "could not find file for ranking expression '%s' (ref:'%s')", - entry.name.c_str(), entry.fileref.c_str()); - } - } - return expressions; -} - OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCfg) { OnnxModels::Vector model_list; for (const auto &entry: modelsCfg.model) { @@ -81,7 +64,7 @@ OnnxModels make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupC model_list.emplace_back(entry.name, file.value()); OnnxModels::configure(entry, model_list.back()); } else { - LOG(warning, "could not find file for onnx model '%s' (ref:'%s')", + LOG(warning, "could not find file for onnx model '%s' (ref:'%s')\n", entry.name.c_str(), entry.fileref.c_str()); } } @@ -94,15 +77,13 @@ public: bool verify(const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &repo, - const RankingExpressions &expressions, - const OnnxModels &models); + OnnxModels models); bool verifyConfig(const VerifyRanksetupConfig &myCfg, const RankProfilesConfig &rankCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, const RankingConstantsConfig &constantsCfg, - const RankingExpressionsConfig &expressionsCfg, const OnnxModelsConfig &modelsCfg); int usage(); @@ -131,10 +112,9 @@ bool App::verify(const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &repo, - const RankingExpressions &expressions, - const OnnxModels &models) + OnnxModels models) { - proton::matching::IndexEnvironment indexEnv(0, schema, props, repo, expressions, models); + proton::matching::IndexEnvironment indexEnv(0, schema, props, repo, models); search::fef::BlueprintFactory factory; search::features::setup_search_features(factory); search::fef::test::setup_fef_test_plugin(factory); @@ -163,7 +143,6 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, const IndexschemaConfig &schemaCfg, const AttributesConfig &attributeCfg, const RankingConstantsConfig &constantsCfg, - const RankingExpressionsConfig &expressionsCfg, const OnnxModelsConfig &modelsCfg) { bool ok = true; @@ -171,7 +150,6 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, search::index::SchemaBuilder::build(schemaCfg, schema); search::index::SchemaBuilder::build(attributeCfg, schema); DummyConstantValueRepo repo(constantsCfg); - auto expressions = make_expressions(expressionsCfg, myCfg); auto models = make_models(modelsCfg, myCfg); for(size_t i = 0; i < rankCfg.rankprofile.size(); i++) { search::fef::Properties properties; @@ -180,7 +158,7 @@ App::verifyConfig(const VerifyRanksetupConfig &myCfg, properties.add(profile.fef.property[j].name, profile.fef.property[j].value); } - if (verify(schema, properties, repo, expressions, models)) { + if (verify(schema, properties, repo, models)) { LOG(info, "rank profile '%s': pass", profile.name.c_str()); } else { LOG(error, "rank profile '%s': FAIL", profile.name.c_str()); @@ -218,7 +196,6 @@ App::Main() ConfigHandle<AttributesConfig>::UP attributesHandle = subscriber.subscribe<AttributesConfig>(cfgId); ConfigHandle<IndexschemaConfig>::UP schemaHandle = subscriber.subscribe<IndexschemaConfig>(cfgId); ConfigHandle<RankingConstantsConfig>::UP constantsHandle = subscriber.subscribe<RankingConstantsConfig>(cfgId); - ConfigHandle<RankingExpressionsConfig>::UP expressionsHandle = subscriber.subscribe<RankingExpressionsConfig>(cfgId); ConfigHandle<OnnxModelsConfig>::UP modelsHandle = subscriber.subscribe<OnnxModelsConfig>(cfgId); subscriber.nextConfig(); @@ -227,7 +204,6 @@ App::Main() *schemaHandle->getConfig(), *attributesHandle->getConfig(), *constantsHandle->getConfig(), - *expressionsHandle->getConfig(), *modelsHandle->getConfig()); } catch (ConfigRuntimeException & e) { LOG(error, "Unable to subscribe to config: %s", e.getMessage().c_str()); diff --git a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp index e4dd4715ed1..6e346bcfa60 100644 --- a/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp +++ b/searchcore/src/apps/vespa-feed-bm/vespa_feed_bm.cpp @@ -187,7 +187,6 @@ std::shared_ptr<DocumentDBConfig> make_document_db_config(std::shared_ptr<Docume 1, std::make_shared<RankProfilesConfig>(), std::make_shared<matching::RankingConstants>(), - std::make_shared<matching::RankingExpressions>(), std::make_shared<matching::OnnxModels>(), indexschema, attributes, diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 3f65f61cbaa..7ae828241c8 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -643,7 +643,6 @@ TEST("require that maintenance controller should change if some config has chang TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankProfilesChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankingConstantsChanged(true))); - TEST_DO(assertMaintenanceControllerShouldChange(CCR().setRankingExpressionsChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setIndexschemaChanged(true))); TEST_DO(assertMaintenanceControllerShouldChange(CCR().setAttributesChanged(true))); @@ -691,7 +690,6 @@ TEST("require that subdbs should change if relevant config changed") TEST_DO(assertSubDbsShouldChange(CCR().setVisibilityDelayChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setRankProfilesChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setRankingConstantsChanged(true))); - TEST_DO(assertSubDbsShouldChange(CCR().setRankingExpressionsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setOnnxModelsChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().setSchemaChanged(true))); TEST_DO(assertSubDbsShouldChange(CCR().set_alloc_config_changed(true))); diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index 544ff853384..aed01ca0192 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -17,7 +17,6 @@ using namespace search::index; using namespace search; using namespace vespa::config::search; using proton::matching::RankingConstants; -using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using std::make_shared; using std::shared_ptr; @@ -70,11 +69,6 @@ public: _builder.rankingConstants(make_shared<RankingConstants>(constants)); return *this; } - MyConfigBuilder &addRankingExpression() { - auto expr_list = RankingExpressions().add("my_expr", "my_file"); - _builder.rankingExpressions(make_shared<RankingExpressions>(expr_list)); - return *this; - } MyConfigBuilder &addOnnxModel() { OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; _builder.onnxModels(make_shared<OnnxModels>(models)); @@ -144,7 +138,6 @@ struct Fixture { fullCfg = MyConfigBuilder(4, schema, repo).addAttribute(). addRankProfile(). addRankingConstant(). - addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(true). @@ -180,7 +173,6 @@ struct DelayAttributeAspectFixture { attrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(true)).addAttribute(). addRankProfile(). addRankingConstant(). - addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(true). @@ -188,7 +180,6 @@ struct DelayAttributeAspectFixture { build(); noAttrCfg = MyConfigBuilder(4, schema, makeDocTypeRepo(hasDocField)).addRankProfile(). addRankingConstant(). - addRankingExpression(). addOnnxModel(). addImportedField(). addSummary(hasDocField). diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index 4ac66e464e2..e6bcbf18495 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -28,7 +28,6 @@ using namespace vespa::config::search; using namespace std::chrono_literals; using vespa::config::content::core::BucketspacesConfig; using proton::matching::RankingConstants; -using proton::matching::RankingExpressions; using proton::matching::OnnxModels; typedef DocumentDBConfigHelper DBCM; @@ -77,10 +76,8 @@ assertEqualSnapshot(const DocumentDBConfig &exp, const DocumentDBConfig &act) { EXPECT_TRUE(exp.getRankProfilesConfig() == act.getRankProfilesConfig()); EXPECT_TRUE(exp.getRankingConstants() == act.getRankingConstants()); - EXPECT_TRUE(exp.getRankingExpressions() == act.getRankingExpressions()); EXPECT_TRUE(exp.getOnnxModels() == act.getOnnxModels()); EXPECT_EQUAL(0u, exp.getRankingConstants().size()); - EXPECT_EQUAL(0u, exp.getRankingExpressions().size()); EXPECT_EQUAL(0u, exp.getOnnxModels().size()); EXPECT_TRUE(exp.getIndexschemaConfig() == act.getIndexschemaConfig()); EXPECT_TRUE(exp.getAttributesConfig() == act.getAttributesConfig()); @@ -111,9 +108,6 @@ addConfigsThatAreNotSavedToDisk(const DocumentDBConfig &cfg) RankingConstants::Vector constants = {{"my_name", "my_type", "my_path"}}; builder.rankingConstants(std::make_shared<RankingConstants>(constants)); - auto expr_list = RankingExpressions().add("my_expr", "my_file"); - builder.rankingExpressions(std::make_shared<RankingExpressions>(expr_list)); - OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; builder.onnxModels(std::make_shared<OnnxModels>(models)); diff --git a/searchcore/src/tests/proton/matching/index_environment/.gitattributes b/searchcore/src/tests/proton/matching/index_environment/.gitattributes deleted file mode 100644 index c7aff0d0972..00000000000 --- a/searchcore/src/tests/proton/matching/index_environment/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -/my_expr.lz4 binary diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index 20b9e1206c7..421ebffafa4 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -3,7 +3,6 @@ #include <vespa/eval/eval/value_cache/constant_value.h> #include <vespa/searchcore/proton/matching/indexenvironment.h> -#include <vespa/searchcore/proton/matching/ranking_expressions.h> using namespace proton::matching; using search::fef::FieldInfo; @@ -16,21 +15,6 @@ using search::index::schema::DataType; using vespalib::eval::ConstantValue; using SIAF = Schema::ImportedAttributeField; -const vespalib::string my_expr_ref( - "this is my reference ranking expression.\n" - "this is my reference ranking expression.\n" - "it will not compile into a function.\n" - "it will not compile into a function.\n" - "it is just some text, that can also be compressed...\n" - "it is just some text, that can also be compressed...\n"); - -RankingExpressions make_expressions() { - RankingExpressions expr_list; - expr_list.add("expr1", TEST_PATH("my_expr")); - expr_list.add("expr2", TEST_PATH("my_expr.lz4")); - return expr_list; -} - OnnxModels make_models() { OnnxModels::Vector list; list.emplace_back(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1")); @@ -66,7 +50,7 @@ struct Fixture { Fixture(Schema::UP schema_) : repo(), schema(std::move(schema_)), - env(7, *schema, Properties(), repo, make_expressions(), make_models()) + env(7, *schema, Properties(), repo, make_models()) { } const FieldInfo *assertField(size_t idx, @@ -139,13 +123,4 @@ TEST_F("require that onnx model config can be obtained", Fixture(buildEmptySchem EXPECT_TRUE(f1.env.getOnnxModel("model3") == nullptr); } -TEST_F("require that external ranking expressions can be obtained", Fixture(buildEmptySchema())) { - auto expr1 = f1.env.getRankingExpression("expr1"); - auto expr2 = f1.env.getRankingExpression("expr2"); - auto expr3 = f1.env.getRankingExpression("expr3"); - EXPECT_EQUAL(expr1, my_expr_ref); - EXPECT_EQUAL(expr2, my_expr_ref); - EXPECT_TRUE(expr3.empty()); -} - TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchcore/src/tests/proton/matching/index_environment/my_expr b/searchcore/src/tests/proton/matching/index_environment/my_expr deleted file mode 100644 index 2fdd1e4c9ff..00000000000 --- a/searchcore/src/tests/proton/matching/index_environment/my_expr +++ /dev/null @@ -1,6 +0,0 @@ -this is my reference ranking expression. -this is my reference ranking expression. -it will not compile into a function. -it will not compile into a function. -it is just some text, that can also be compressed... -it is just some text, that can also be compressed... diff --git a/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 b/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 Binary files differdeleted file mode 100644 index c88594ce81d..00000000000 --- a/searchcore/src/tests/proton/matching/index_environment/my_expr.lz4 +++ /dev/null diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index 2bee53a5215..8057dbf2da6 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -280,7 +280,7 @@ struct MyWorld { } Matcher::SP createMatcher() { - return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), 0); + return std::make_shared<Matcher>(schema, config, clock, queryLimiter, constantValueRepo, OnnxModels(), 0); } struct MySearchHandler : ISearchHandler { diff --git a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp index 20a10aee678..e42f8323473 100644 --- a/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp +++ b/searchcore/src/tests/proton/proton_config_fetcher/proton_config_fetcher_test.cpp @@ -10,7 +10,6 @@ #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/proton/common/subdbtype.h> #include <vespa/searchcore/config/config-ranking-constants.h> -#include <vespa/searchcore/config/config-ranking-expressions.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/document/repo/documenttyperepo.h> @@ -53,7 +52,6 @@ struct DoctypeFixture { AttributesConfigBuilder attributesBuilder; RankProfilesConfigBuilder rankProfilesBuilder; RankingConstantsConfigBuilder rankingConstantsBuilder; - RankingExpressionsConfigBuilder rankingExpressionsBuilder; OnnxModelsConfigBuilder onnxModelsBuilder; IndexschemaConfigBuilder indexschemaBuilder; SummaryConfigBuilder summaryBuilder; @@ -110,7 +108,6 @@ struct ConfigTestFixture { set.addBuilder(db.configid, &fixture->attributesBuilder); set.addBuilder(db.configid, &fixture->rankProfilesBuilder); set.addBuilder(db.configid, &fixture->rankingConstantsBuilder); - set.addBuilder(db.configid, &fixture->rankingExpressionsBuilder); set.addBuilder(db.configid, &fixture->onnxModelsBuilder); set.addBuilder(db.configid, &fixture->indexschemaBuilder); set.addBuilder(db.configid, &fixture->summaryBuilder); @@ -265,7 +262,7 @@ TEST_FF("require that documentdb config manager subscribes for config", DocumentDBConfigManager(f1.configId + "/typea", "typea")) { f1.addDocType("typea"); const ConfigKeySet keySet(f2.createConfigKeySet()); - ASSERT_EQUAL(10u, keySet.size()); + ASSERT_EQUAL(9u, keySet.size()); ASSERT_TRUE(f1.configEqual("typea", getDocumentDBConfig(f1, f2))); } diff --git a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp index faf09be3bbd..16e2fc4b1a8 100644 --- a/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp +++ b/searchcore/src/tests/proton/proton_configurer/proton_configurer_test.cpp @@ -48,13 +48,14 @@ using std::map; using search::index::Schema; using search::index::SchemaBuilder; using proton::matching::RankingConstants; -using proton::matching::RankingExpressions; using proton::matching::OnnxModels; struct DBConfigFixture { using UP = std::unique_ptr<DBConfigFixture>; AttributesConfigBuilder _attributesBuilder; RankProfilesConfigBuilder _rankProfilesBuilder; + RankingConstantsConfigBuilder _rankingConstantsBuilder; + OnnxModelsConfigBuilder _onnxModelsBuilder; IndexschemaConfigBuilder _indexschemaBuilder; SummaryConfigBuilder _summaryBuilder; SummarymapConfigBuilder _summarymapBuilder; @@ -75,11 +76,6 @@ struct DBConfigFixture { return std::make_shared<RankingConstants>(); } - RankingExpressions::SP buildRankingExpressions() - { - return std::make_shared<RankingExpressions>(); - } - OnnxModels::SP buildOnnxModels() { return std::make_shared<OnnxModels>(); @@ -95,7 +91,6 @@ struct DBConfigFixture { (generation, std::make_shared<RankProfilesConfig>(_rankProfilesBuilder), buildRankingConstants(), - buildRankingExpressions(), buildOnnxModels(), std::make_shared<IndexschemaConfig>(_indexschemaBuilder), std::make_shared<AttributesConfig>(_attributesBuilder), diff --git a/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression b/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression deleted file mode 100644 index 475b311dc84..00000000000 --- a/searchcore/src/tests/proton/verify_ranksetup/bad_ranking_expression +++ /dev/null @@ -1 +0,0 @@ -join(constant(my_tensor),attribute(date),f(t,d)(join(t,d,f(x,y)(x+y)))) diff --git a/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression b/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression deleted file mode 100644 index f6712be7a82..00000000000 --- a/searchcore/src/tests/proton/verify_ranksetup/good_ranking_expression +++ /dev/null @@ -1 +0,0 @@ -join(constant(my_tensor),attribute(date),f(t,d)(t+d)) diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 31557f13a54..c46990732b7 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -71,7 +71,6 @@ struct Setup { std::map<std::string,std::string> properties; std::map<std::string,std::string> constants; std::vector<bool> extra_profiles; - std::map<std::string,std::string> ranking_expressions; std::map<std::string,OnnxModel> onnx_models; Setup(); ~Setup(); @@ -97,9 +96,6 @@ struct Setup { void rank_expr(const std::string &name, const std::string &expr) { property(fmt("rankingExpression(%s).rankingScript", name.c_str()), expr); } - void ext_rank_expr(const std::string &name, const std::string &file) { - ranking_expressions.insert_or_assign(name, TEST_PATH(file)); - } void first_phase(const std::string &feature) { property(rank::FirstPhase::NAME, feature); } @@ -161,14 +157,6 @@ struct Setup { ++idx; } } - void write_ranking_expressions(const Writer &out) { - size_t idx = 0; - for (const auto &entry: ranking_expressions) { - out.fmt("expression[%zu].name \"%s\"\n", idx, entry.first.c_str()); - out.fmt("expression[%zu].fileref \"expr_ref_%zu\"\n", idx, idx); - ++idx; - } - } void write_onnx_models(const Writer &out) { size_t idx = 0; for (const auto &entry: onnx_models) { @@ -191,12 +179,6 @@ struct Setup { } void write_self_cfg(const Writer &out) { size_t idx = 0; - for (const auto &entry: ranking_expressions) { - out.fmt("file[%zu].ref \"expr_ref_%zu\"\n", idx, idx); - out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.c_str()); - ++idx; - } - idx = 0; for (const auto &entry: onnx_models) { out.fmt("file[%zu].ref \"onnx_ref_%zu\"\n", idx, idx); out.fmt("file[%zu].path \"%s\"\n", idx, entry.second.file_path().c_str()); @@ -208,7 +190,6 @@ struct Setup { write_indexschema(Writer(gen_dir + "/indexschema.cfg")); write_rank_profiles(Writer(gen_dir + "/rank-profiles.cfg")); write_ranking_constants(Writer(gen_dir + "/ranking-constants.cfg")); - write_ranking_expressions(Writer(gen_dir + "/ranking-expressions.cfg")); write_onnx_models(Writer(gen_dir + "/onnx-models.cfg")); write_self_cfg(Writer(gen_dir + "/verify-ranksetup.cfg")); } @@ -377,23 +358,6 @@ TEST_F("require that imported attribute field can be used by rank feature", Simp //----------------------------------------------------------------------------- -TEST_F("require that external ranking expression can be verified", SimpleSetup()) { - f.ext_rank_expr("my_expr", "good_ranking_expression"); - f.verify_valid({"rankingExpression(my_expr)"}); -} - -TEST_F("require that external ranking expression can fail verification", SimpleSetup()) { - f.ext_rank_expr("my_expr", "bad_ranking_expression"); - f.verify_invalid({"rankingExpression(my_expr)"}); -} - -TEST_F("require that missing expression file fails verification", SimpleSetup()) { - f.ext_rank_expr("my_expr", "missing_ranking_expression_file"); - f.verify_invalid({"rankingExpression(my_expr)"}); -} - -//----------------------------------------------------------------------------- - TEST_F("require that onnx model can be verified", OnnxSetup()) { f.rank_expr("query_tensor", "tensor<float>(a[1],b[4]):[[1,2,3,4]]"); f.rank_expr("attribute_tensor", "tensor<float>(a[4],b[1]):[[5],[6],[7],[8]]"); diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index d83d8ed91d9..a4688b5fdca 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -27,7 +27,6 @@ vespa_add_library(searchcore_matching STATIC querylimiter.cpp querynodes.cpp ranking_constants.cpp - ranking_expressions.cpp requestcontext.cpp resolveviewvisitor.cpp result_processor.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp index 5f5cc4dcf76..013f359c4f9 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp @@ -65,17 +65,15 @@ IndexEnvironment::IndexEnvironment(uint32_t distributionKey, const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &constantValueRepo, - RankingExpressions rankingExpressions, OnnxModels onnxModels) - : _tableManager(), - _properties(props), - _fieldNames(), - _fields(), - _motivation(UNKNOWN), - _constantValueRepo(constantValueRepo), - _rankingExpressions(std::move(rankingExpressions)), - _onnxModels(std::move(onnxModels)), - _distributionKey(distributionKey) + : _tableManager(), + _properties(props), + _fieldNames(), + _fields(), + _motivation(UNKNOWN), + _constantValueRepo(constantValueRepo), + _onnxModels(std::move(onnxModels)), + _distributionKey(distributionKey) { _tableManager.addFactory(std::make_shared<search::fef::FunctionTableFactory>(256)); extractFields(schema); @@ -133,12 +131,6 @@ IndexEnvironment::hintFieldAccess(uint32_t ) const { } void IndexEnvironment::hintAttributeAccess(const string &) const { } -vespalib::string -IndexEnvironment::getRankingExpression(const vespalib::string &name) const -{ - return _rankingExpressions.loadExpression(name); -} - const search::fef::OnnxModel * IndexEnvironment::getOnnxModel(const vespalib::string &name) const { diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h index d270825d3f7..ad51eb17b4d 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h @@ -3,7 +3,6 @@ #pragma once #include "onnx_models.h" -#include "ranking_expressions.h" #include "i_constant_value_repo.h" #include <vespa/searchlib/fef/fieldinfo.h> #include <vespa/searchlib/fef/iindexenvironment.h> @@ -27,7 +26,6 @@ private: std::vector<search::fef::FieldInfo> _fields; mutable FeatureMotivation _motivation; const IConstantValueRepo &_constantValueRepo; - RankingExpressions _rankingExpressions; OnnxModels _onnxModels; uint32_t _distributionKey; @@ -48,14 +46,12 @@ public: * @param schema the index schema * @param props config * @param constantValueRepo repo used to access constant values for ranking - * @param rankingExpressions processed config about ranking expressions * @param onnxModels processed config about onnx models **/ IndexEnvironment(uint32_t distributionKey, const search::index::Schema &schema, const search::fef::Properties &props, const IConstantValueRepo &constantValueRepo, - RankingExpressions rankingExpressions, OnnxModels onnxModels); const search::fef::Properties &getProperties() const override; @@ -72,7 +68,6 @@ public: vespalib::eval::ConstantValue::UP getConstantValue(const vespalib::string &name) const override { return _constantValueRepo.getConstant(name); } - vespalib::string getRankingExpression(const vespalib::string &name) const override; const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override; ~IndexEnvironment() override; diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 806753a6660..e459a45040b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -99,18 +99,17 @@ handleGroupingSession(SessionManager &sessionMgr, GroupingContext & groupingCont } // namespace proton::matching::<unnamed> Matcher::Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, - QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, - RankingExpressions rankingExpressions, OnnxModels onnxModels, uint32_t distributionKey) - : _indexEnv(distributionKey, schema, props, constantValueRepo, std::move(rankingExpressions), std::move(onnxModels)), - _blueprintFactory(), - _rankSetup(), - _viewResolver(ViewResolver::createFromSchema(schema)), - _statsLock(), - _stats(), - _startTime(my_clock::now()), - _clock(clock), - _queryLimiter(queryLimiter), - _distributionKey(distributionKey) + QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, OnnxModels onnxModels, uint32_t distributionKey) + : _indexEnv(distributionKey, schema, props, constantValueRepo, std::move(onnxModels)), + _blueprintFactory(), + _rankSetup(), + _viewResolver(ViewResolver::createFromSchema(schema)), + _statsLock(), + _stats(), + _startTime(my_clock::now()), + _clock(clock), + _queryLimiter(queryLimiter), + _distributionKey(distributionKey) { search::features::setup_search_features(_blueprintFactory); search::fef::test::setup_fef_test_plugin(_blueprintFactory); diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index d40ae9b18f1..dcd1bbb2b46 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -88,8 +88,7 @@ public: **/ Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, QueryLimiter &queryLimiter, - const IConstantValueRepo &constantValueRepo, - RankingExpressions rankingExpressions, OnnxModels onnxModels, + const IConstantValueRepo &constantValueRepo, OnnxModels onnxModels, uint32_t distributionKey); const search::fef::IIndexEnvironment &get_index_env() const { return _indexEnv; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp deleted file mode 100644 index 9b74f76aa6e..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "ranking_expressions.h" -#include <vespa/vespalib/io/mapped_file_input.h> -#include <vespa/vespalib/data/lz4_input_decoder.h> -#include <vespa/vespalib/util/size_literals.h> - -#include <vespa/log/log.h> -LOG_SETUP(".proton.matching.ranking_expressions"); - -namespace proton::matching { - -namespace { - -vespalib::string extract_data(vespalib::Input &input) { - vespalib::string result; - for (auto chunk = input.obtain(); chunk.size > 0; chunk = input.obtain()) { - result.append(vespalib::stringref(chunk.data, chunk.size)); - input.evict(chunk.size); - } - return result; -} - -} // unnamed - -RankingExpressions::RankingExpressions() = default; -RankingExpressions::RankingExpressions(RankingExpressions &&rhs) = default; -RankingExpressions::RankingExpressions(const RankingExpressions &rhs) = default; -RankingExpressions::~RankingExpressions() = default; - -RankingExpressions & -RankingExpressions::add(const vespalib::string &name, const vespalib::string &path) -{ - _expressions.insert_or_assign(name, path); - return *this; -} - -vespalib::string -RankingExpressions::loadExpression(const vespalib::string &name) const -{ - auto pos = _expressions.find(name); - if (pos == _expressions.end()) { - // not warning about missing expression here since what we - // think is a name might be an expression itself. - return {}; - } - auto path = pos->second; - vespalib::MappedFileInput file(path); - if (!file.valid()) { - LOG(warning, "could not read file: %s", path.c_str()); - return {}; - } - if (ends_with(path, ".lz4")) { - size_t buffer_size = 64_Ki; - vespalib::Lz4InputDecoder lz4_decoder(file, buffer_size); - auto result = extract_data(lz4_decoder); - if (lz4_decoder.failed()) { - LOG(warning, "file contains lz4 errors (%s): %s", - lz4_decoder.reason().c_str(), path.c_str()); - return {}; - } - return result; - } - return extract_data(file); -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h deleted file mode 100644 index e1472b03ee0..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright Verizon Media. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include <vespa/vespalib/stllike/string.h> -#include <memory> -#include <map> - -namespace proton::matching { - -/** - * Class representing a collection of named ranking expressions - * obtained through file-distribution. - */ -class RankingExpressions -{ -private: - // expression name -> full_path of expression file - std::map<vespalib::string,vespalib::string> _expressions; - -public: - RankingExpressions(); - RankingExpressions(RankingExpressions &&rhs); - RankingExpressions(const RankingExpressions &rhs); - ~RankingExpressions(); - using SP = std::shared_ptr<RankingExpressions>; - bool operator==(const RankingExpressions &rhs) const { - return _expressions == rhs._expressions; - } - size_t size() const { return _expressions.size(); } - RankingExpressions &add(const vespalib::string &name, const vespalib::string &path); - vespalib::string loadExpression(const vespalib::string &name) const; -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp index e39188042d7..7c487043b5b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.cpp @@ -35,7 +35,6 @@ namespace proton { DocumentDBConfig::ComparisonResult::ComparisonResult() : rankProfilesChanged(false), rankingConstantsChanged(false), - rankingExpressionsChanged(false), onnxModelsChanged(false), indexschemaChanged(false), attributesChanged(false), @@ -59,7 +58,6 @@ DocumentDBConfig::DocumentDBConfig( int64_t generation, const RankProfilesConfigSP &rankProfiles, const RankingConstants::SP &rankingConstants, - const RankingExpressions::SP &rankingExpressions, const OnnxModels::SP &onnxModels, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, @@ -82,7 +80,6 @@ DocumentDBConfig::DocumentDBConfig( _generation(generation), _rankProfiles(rankProfiles), _rankingConstants(rankingConstants), - _rankingExpressions(rankingExpressions), _onnxModels(onnxModels), _indexschema(indexschema), _attributes(attributes), @@ -110,7 +107,6 @@ DocumentDBConfig(const DocumentDBConfig &cfg) _generation(cfg._generation), _rankProfiles(cfg._rankProfiles), _rankingConstants(cfg._rankingConstants), - _rankingExpressions(cfg._rankingExpressions), _onnxModels(cfg._onnxModels), _indexschema(cfg._indexschema), _attributes(cfg._attributes), @@ -137,7 +133,6 @@ DocumentDBConfig::operator==(const DocumentDBConfig & rhs) const { return equals<RankProfilesConfig>(_rankProfiles.get(), rhs._rankProfiles.get()) && equals<RankingConstants>(_rankingConstants.get(), rhs._rankingConstants.get()) && - equals<RankingExpressions>(_rankingExpressions.get(), rhs._rankingExpressions.get()) && equals<OnnxModels>(_onnxModels.get(), rhs._onnxModels.get()) && equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get()) && equals<AttributesConfig>(_attributes.get(), rhs._attributes.get()) && @@ -162,7 +157,6 @@ DocumentDBConfig::compare(const DocumentDBConfig &rhs) const ComparisonResult retval; retval.rankProfilesChanged = !equals<RankProfilesConfig>(_rankProfiles.get(), rhs._rankProfiles.get()); retval.rankingConstantsChanged = !equals<RankingConstants>(_rankingConstants.get(), rhs._rankingConstants.get()); - retval.rankingExpressionsChanged = !equals<RankingExpressions>(_rankingExpressions.get(), rhs._rankingExpressions.get()); retval.onnxModelsChanged = !equals<OnnxModels>(_onnxModels.get(), rhs._onnxModels.get()); retval.indexschemaChanged = !equals<IndexschemaConfig>(_indexschema.get(), rhs._indexschema.get()); retval.attributesChanged = !equals<AttributesConfig>(_attributes.get(), rhs._attributes.get()); @@ -189,7 +183,6 @@ DocumentDBConfig::valid() const { return _rankProfiles && _rankingConstants && - _rankingExpressions && _onnxModels && _indexschema && _attributes && @@ -233,7 +226,6 @@ DocumentDBConfig::makeReplayConfig(const SP & orig) o._generation, emptyConfig(o._rankProfiles), std::make_shared<RankingConstants>(), - std::make_shared<RankingExpressions>(), std::make_shared<OnnxModels>(), o._indexschema, o._attributes, @@ -277,7 +269,6 @@ DocumentDBConfig::newFromAttributesConfig(const AttributesConfigSP &attributes) _generation, _rankProfiles, _rankingConstants, - _rankingExpressions, _onnxModels, _indexschema, attributes, @@ -316,7 +307,6 @@ DocumentDBConfig::makeDelayedAttributeAspectConfig(const SP &newCfg, const Docum (n._generation, n._rankProfiles, n._rankingConstants, - n._rankingExpressions, n._onnxModels, n._indexschema, attributeAspectDelayer.getAttributesConfig(), diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h index 2ca7fb1dfa8..8e24ed8e96a 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfig.h @@ -6,7 +6,6 @@ #include <vespa/searchlib/common/tunefileinfo.h> #include <vespa/searchcommon/common/schema.h> #include <vespa/searchcore/proton/matching/ranking_constants.h> -#include <vespa/searchcore/proton/matching/ranking_expressions.h> #include <vespa/searchcore/proton/matching/onnx_models.h> #include <vespa/config/retriever/configkeyset.h> #include <vespa/config/retriever/configsnapshot.h> @@ -41,7 +40,6 @@ public: public: bool rankProfilesChanged; bool rankingConstantsChanged; - bool rankingExpressionsChanged; bool onnxModelsChanged; bool indexschemaChanged; bool attributesChanged; @@ -63,7 +61,6 @@ public: ComparisonResult(); ComparisonResult &setRankProfilesChanged(bool val) { rankProfilesChanged = val; return *this; } ComparisonResult &setRankingConstantsChanged(bool val) { rankingConstantsChanged = val; return *this; } - ComparisonResult &setRankingExpressionsChanged(bool val) { rankingExpressionsChanged = val; return *this; } ComparisonResult &setOnnxModelsChanged(bool val) { onnxModelsChanged = val; return *this; } ComparisonResult &setIndexschemaChanged(bool val) { indexschemaChanged = val; return *this; } ComparisonResult &setAttributesChanged(bool val) { attributesChanged = val; return *this; } @@ -104,7 +101,6 @@ public: using RankProfilesConfig = const vespa::config::search::internal::InternalRankProfilesType; using RankProfilesConfigSP = std::shared_ptr<RankProfilesConfig>; using RankingConstants = matching::RankingConstants; - using RankingExpressions = matching::RankingExpressions; using OnnxModels = matching::OnnxModels; using SummaryConfig = const vespa::config::search::internal::InternalSummaryType; using SummaryConfigSP = std::shared_ptr<SummaryConfig>; @@ -124,7 +120,6 @@ private: int64_t _generation; RankProfilesConfigSP _rankProfiles; RankingConstants::SP _rankingConstants; - RankingExpressions::SP _rankingExpressions; OnnxModels::SP _onnxModels; IndexschemaConfigSP _indexschema; AttributesConfigSP _attributes; @@ -164,7 +159,6 @@ public: DocumentDBConfig(int64_t generation, const RankProfilesConfigSP &rankProfiles, const RankingConstants::SP &rankingConstants, - const RankingExpressions::SP &rankingExpressions, const OnnxModels::SP &onnxModels, const IndexschemaConfigSP &indexschema, const AttributesConfigSP &attributes, @@ -195,7 +189,6 @@ public: const RankProfilesConfig &getRankProfilesConfig() const { return *_rankProfiles; } const RankingConstants &getRankingConstants() const { return *_rankingConstants; } - const RankingExpressions &getRankingExpressions() const { return *_rankingExpressions; } const OnnxModels &getOnnxModels() const { return *_onnxModels; } const IndexschemaConfig &getIndexschemaConfig() const { return *_indexschema; } const AttributesConfig &getAttributesConfig() const { return *_attributes; } @@ -205,7 +198,6 @@ public: const DocumenttypesConfig &getDocumenttypesConfig() const { return *_documenttypes; } const RankProfilesConfigSP &getRankProfilesConfigSP() const { return _rankProfiles; } const RankingConstants::SP &getRankingConstantsSP() const { return _rankingConstants; } - const RankingExpressions::SP &getRankingExpressionsSP() const { return _rankingExpressions; } const OnnxModels::SP &getOnnxModelsSP() const { return _onnxModels; } const IndexschemaConfigSP &getIndexschemaConfigSP() const { return _indexschema; } const AttributesConfigSP &getAttributesConfigSP() const { return _attributes; } diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index fb154081963..8d513fde62f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -6,7 +6,6 @@ #include <vespa/searchcore/proton/common/alloc_config.h> #include <vespa/searchcore/proton/common/hw_info.h> #include <vespa/searchcore/config/config-ranking-constants.h> -#include <vespa/searchcore/config/config-ranking-expressions.h> #include <vespa/searchcore/config/config-onnx-models.h> #include <vespa/config-imported-fields.h> #include <vespa/config-rank-profiles.h> @@ -21,7 +20,6 @@ #include <vespa/searchlib/index/schemautil.h> #include <vespa/searchsummary/config/config-juniperrc.h> #include <vespa/vespalib/time/time_box.h> -#include <vespa/vespalib/util/stringfmt.h> #include <thread> #include <cassert> @@ -38,7 +36,6 @@ using search::TuneFileDocumentDB; using search::index::Schema; using search::index::SchemaBuilder; using proton::matching::RankingConstants; -using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using vespalib::compression::CompressionConfig; using search::LogDocumentStore; @@ -48,8 +45,6 @@ using search::WriteableFileChunk; using std::make_shared; using std::make_unique; -using vespalib::make_string_short::fmt; - namespace proton { const ConfigKeySet @@ -58,7 +53,6 @@ DocumentDBConfigManager::createConfigKeySet() const ConfigKeySet set; set.add<RankProfilesConfig, RankingConstantsConfig, - RankingExpressionsConfig, OnnxModelsConfig, IndexschemaConfig, AttributesConfig, @@ -270,21 +264,6 @@ build_alloc_config(const ProtonConfig& proton_config, const vespalib::string& do distribution_config.redundancy, distribution_config.searchablecopies); } -vespalib::string resolve_file(config::RpcFileAcquirer &fileAcquirer, vespalib::TimeBox &timeBox, - const vespalib::string &desc, const vespalib::string &fileref) -{ - vespalib::string filePath; - LOG(info, "Waiting for file acquirer (%s, ref='%s')", desc.c_str(), fileref.c_str()); - while (timeBox.hasTimeLeft() && (filePath == "")) { - filePath = fileAcquirer.wait_for(fileref, timeBox.timeLeft()); - if (filePath == "") { - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - } - LOG(info, "Got file path from file acquirer: '%s' (%s, ref='%s')", filePath.c_str(), desc.c_str(), fileref.c_str()); - return filePath; -} - } void @@ -292,7 +271,6 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) { using RankProfilesConfigSP = DocumentDBConfig::RankProfilesConfigSP; using RankingConstantsConfigSP = std::shared_ptr<vespa::config::search::core::RankingConstantsConfig>; - using RankingExpressionsConfigSP = std::shared_ptr<vespa::config::search::core::RankingExpressionsConfig>; using OnnxModelsConfigSP = std::shared_ptr<vespa::config::search::core::OnnxModelsConfig>; using IndexschemaConfigSP = DocumentDBConfig::IndexschemaConfigSP; using SummaryConfigSP = DocumentDBConfig::SummaryConfigSP; @@ -304,7 +282,6 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) DocumentDBConfig::SP current = _pendingConfigSnapshot; RankProfilesConfigSP newRankProfilesConfig; matching::RankingConstants::SP newRankingConstants; - matching::RankingExpressions::SP newRankingExpressions; matching::OnnxModels::SP newOnnxModels; IndexschemaConfigSP newIndexschemaConfig; MaintenanceConfigSP oldMaintenanceConfig; @@ -331,7 +308,6 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) if (current) { newRankProfilesConfig = current->getRankProfilesConfigSP(); newRankingConstants = current->getRankingConstantsSP(); - newRankingExpressions = current->getRankingExpressionsSP(); newOnnxModels = current->getOnnxModelsSP(); newIndexschemaConfig = current->getIndexschemaConfigSP(); oldMaintenanceConfig = current->getMaintenanceConfigSP(); @@ -352,29 +328,22 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) config::RpcFileAcquirer fileAcquirer(spec); vespalib::TimeBox timeBox(5*60, 5); for (const RankingConstantsConfig::Constant &rc : newRankingConstantsConfig->constant) { - auto desc = fmt("name='%s', type='%s'", rc.name.c_str(), rc.type.c_str()); - vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); + vespalib::string filePath; + LOG(info, "Waiting for file acquirer (name='%s', type='%s', ref='%s')", + rc.name.c_str(), rc.type.c_str(), rc.fileref.c_str()); + while (timeBox.hasTimeLeft() && (filePath == "")) { + filePath = fileAcquirer.wait_for(rc.fileref, timeBox.timeLeft()); + if (filePath == "") { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + LOG(info, "Got file path from file acquirer: '%s' (name='%s', type='%s', ref='%s')", + filePath.c_str(), rc.name.c_str(), rc.type.c_str(), rc.fileref.c_str()); constants.emplace_back(rc.name, rc.type, filePath); } } newRankingConstants = std::make_shared<RankingConstants>(constants); } - if (snapshot.isChanged<RankingExpressionsConfig>(_configId, currentGeneration)) { - RankingExpressionsConfigSP newRankingExpressionsConfig = RankingExpressionsConfigSP( - snapshot.getConfig<RankingExpressionsConfig>(_configId)); - const vespalib::string &spec = _bootstrapConfig->getFiledistributorrpcConfig().connectionspec; - RankingExpressions expressions; - if (spec != "") { - config::RpcFileAcquirer fileAcquirer(spec); - vespalib::TimeBox timeBox(5*60, 5); - for (const RankingExpressionsConfig::Expression &rc : newRankingExpressionsConfig->expression) { - auto desc = fmt("name='%s'", rc.name.c_str()); - vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); - expressions.add(rc.name, filePath); - } - } - newRankingExpressions = std::make_shared<RankingExpressions>(std::move(expressions)); - } if (snapshot.isChanged<OnnxModelsConfig>(_configId, currentGeneration)) { OnnxModelsConfigSP newOnnxModelsConfig = OnnxModelsConfigSP( snapshot.getConfig<OnnxModelsConfig>(_configId)); @@ -384,8 +353,17 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) config::RpcFileAcquirer fileAcquirer(spec); vespalib::TimeBox timeBox(5*60, 5); for (const OnnxModelsConfig::Model &rc : newOnnxModelsConfig->model) { - auto desc = fmt("name='%s'", rc.name.c_str()); - vespalib::string filePath = resolve_file(fileAcquirer, timeBox, desc, rc.fileref); + vespalib::string filePath; + LOG(info, "Waiting for file acquirer (name='%s', ref='%s')", + rc.name.c_str(), rc.fileref.c_str()); + while (timeBox.hasTimeLeft() && (filePath == "")) { + filePath = fileAcquirer.wait_for(rc.fileref, timeBox.timeLeft()); + if (filePath == "") { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + } + LOG(info, "Got file path from file acquirer: '%s' (name='%s', ref='%s')", + filePath.c_str(), rc.name.c_str(), rc.fileref.c_str()); models.emplace_back(rc.name, filePath); OnnxModels::configure(rc, models.back()); } @@ -425,7 +403,6 @@ DocumentDBConfigManager::update(const ConfigSnapshot &snapshot) auto newSnapshot = std::make_shared<DocumentDBConfig>(generation, newRankProfilesConfig, newRankingConstants, - newRankingExpressions, newOnnxModels, newIndexschemaConfig, filterImportedAttributes(newAttributesConfig), diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp index 7288e3e98fb..53c96a81134 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp @@ -2,13 +2,11 @@ #include "matchers.h" #include <vespa/searchcore/proton/matching/matcher.h> -#include <vespa/searchcore/proton/matching/ranking_expressions.h> #include <vespa/searchcore/proton/matching/onnx_models.h> #include <vespa/vespalib/stllike/hash_map.hpp> namespace proton { -using matching::RankingExpressions; using matching::OnnxModels; Matchers::Matchers(const vespalib::Clock &clock, @@ -16,7 +14,7 @@ Matchers::Matchers(const vespalib::Clock &clock, const matching::IConstantValueRepo &constantValueRepo) : _rpmap(), _fallback(new matching::Matcher(search::index::Schema(), search::fef::Properties(), - clock, queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), -1)), + clock, queryLimiter, constantValueRepo, OnnxModels(), -1)), _default() { } diff --git a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp index 7cdc523e381..bb6c5423175 100644 --- a/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/reconfig_params.cpp @@ -15,7 +15,6 @@ ReconfigParams::configHasChanged() const { return _res.rankProfilesChanged || _res.rankingConstantsChanged || - _res.rankingExpressionsChanged || _res.onnxModelsChanged || _res.indexschemaChanged || _res.attributesChanged || @@ -41,7 +40,7 @@ ReconfigParams::shouldSchemaChange() const bool ReconfigParams::shouldMatchersChange() const { - return _res.rankProfilesChanged || _res.rankingConstantsChanged || _res.rankingExpressionsChanged || _res.onnxModelsChanged || shouldSchemaChange(); + return _res.rankProfilesChanged || _res.rankingConstantsChanged || _res.onnxModelsChanged || shouldSchemaChange(); } bool diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index 05e68ee0ec4..1e5c558af0b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -21,7 +21,6 @@ using vespa::config::search::RankProfilesConfig; namespace proton { using matching::Matcher; -using matching::RankingExpressions; using matching::OnnxModels; typedef AttributeReprocessingInitializer::Config ARIConfig; @@ -108,7 +107,6 @@ SearchableDocSubDBConfigurer::~SearchableDocSubDBConfigurer() = default; Matchers::UP SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, const RankProfilesConfig &cfg, - const RankingExpressions &rankingExpressions, const OnnxModels &onnxModels) { auto newMatchers = std::make_unique<Matchers>(_clock, _queryLimiter, _constantValueRepo); @@ -119,8 +117,7 @@ SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, properties.add(property.name, property.value); } // schema instance only used during call. - auto profptr = std::make_shared<Matcher>(*schema, properties, _clock, _queryLimiter, _constantValueRepo, - rankingExpressions, onnxModels, _distributionKey); + auto profptr = std::make_shared<Matcher>(*schema, properties, _clock, _queryLimiter, _constantValueRepo, onnxModels, _distributionKey); newMatchers->add(name, profptr); } return newMatchers; @@ -188,7 +185,6 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, _constantValueRepo.reconfigure(newConfig.getRankingConstants()); Matchers::SP newMatchers = createMatchers(newConfig.getSchemaSP(), newConfig.getRankProfilesConfig(), - newConfig.getRankingExpressions(), newConfig.getOnnxModels()); matchers = newMatchers; shouldMatchViewChange = true; diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index 1a9ba1abff7..d460bb6506f 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -74,7 +74,6 @@ public: Matchers::UP createMatchers(const search::index::Schema::SP &schema, const vespa::config::search::RankProfilesConfig &cfg, - const proton::matching::RankingExpressions &rankingExpressions, const proton::matching::OnnxModels &onnxModels); void reconfigureIndexSearchable(); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index d7aaa901530..51e6f8e45df 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -201,8 +201,7 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess const Schema::SP &schema = configSnapshot.getSchemaSP(); const IIndexManager::SP &indexMgr = getIndexManager(); _constantValueRepo.reconfigure(configSnapshot.getRankingConstants()); - Matchers::SP matchers = _configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig(), - configSnapshot.getRankingExpressions(), configSnapshot.getOnnxModels()); + Matchers::SP matchers = _configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig(), configSnapshot.getOnnxModels()); auto matchView = std::make_shared<MatchView>(std::move(matchers), indexMgr->getSearchable(), attrMgr, sessionManager, _metaStoreCtx, _docIdLimit); _rSearchView.set(SearchView::create( diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp index 18774e60202..17cb91ac4ce 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.cpp @@ -34,7 +34,6 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(int64_t generation, : _generation(generation), _rankProfiles(std::make_shared<RankProfilesConfig>()), _rankingConstants(std::make_shared<matching::RankingConstants>()), - _rankingExpressions(std::make_shared<matching::RankingExpressions>()), _onnxModels(std::make_shared<matching::OnnxModels>()), _indexschema(std::make_shared<IndexschemaConfig>()), _attributes(std::make_shared<AttributesConfig>()), @@ -59,7 +58,6 @@ DocumentDBConfigBuilder::DocumentDBConfigBuilder(const DocumentDBConfig &cfg) : _generation(cfg.getGeneration()), _rankProfiles(cfg.getRankProfilesConfigSP()), _rankingConstants(cfg.getRankingConstantsSP()), - _rankingExpressions(cfg.getRankingExpressionsSP()), _onnxModels(cfg.getOnnxModelsSP()), _indexschema(cfg.getIndexschemaConfigSP()), _attributes(cfg.getAttributesConfigSP()), @@ -88,7 +86,6 @@ DocumentDBConfigBuilder::build() _generation, _rankProfiles, _rankingConstants, - _rankingExpressions, _onnxModels, _indexschema, _attributes, diff --git a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h index a248ed5544a..706e14e73db 100644 --- a/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h +++ b/searchcore/src/vespa/searchcore/proton/test/documentdb_config_builder.h @@ -14,7 +14,6 @@ private: int64_t _generation; DocumentDBConfig::RankProfilesConfigSP _rankProfiles; DocumentDBConfig::RankingConstants::SP _rankingConstants; - DocumentDBConfig::RankingExpressions::SP _rankingExpressions; DocumentDBConfig::OnnxModels::SP _onnxModels; DocumentDBConfig::IndexschemaConfigSP _indexschema; DocumentDBConfig::AttributesConfigSP _attributes; @@ -58,10 +57,6 @@ public: _rankingConstants = rankingConstants_in; return *this; } - DocumentDBConfigBuilder &rankingExpressions(const DocumentDBConfig::RankingExpressions::SP &rankingExpressions_in) { - _rankingExpressions = rankingExpressions_in; - return *this; - } DocumentDBConfigBuilder &onnxModels(const DocumentDBConfig::OnnxModels::SP &onnxModels_in) { _onnxModels = onnxModels_in; return *this; diff --git a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp index 6baa6581edf..251040ecfa7 100644 --- a/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp +++ b/searchlib/src/tests/features/ranking_expression/ranking_expression_test.cpp @@ -67,38 +67,28 @@ struct SetupResult { RankingExpressionBlueprint rank; DummyDependencyHandler deps; bool setup_ok; - SetupResult(const TypeMap &object_inputs, const vespalib::string &expression, - bool external_expression = false); + SetupResult(const TypeMap &object_inputs, const vespalib::string &expression); ~SetupResult(); }; SetupResult::SetupResult(const TypeMap &object_inputs, - const vespalib::string &expression, - bool external_expression) + const vespalib::string &expression) : stash(), index_env(), query_env(&index_env), rank(make_replacer()), deps(rank), setup_ok(false) { rank.setName("self"); + index_env.getProperties().add("self.rankingScript", expression); for (const auto &input: object_inputs) { deps.define_object_input(input.first, ValueType::from_spec(input.second)); } - std::vector<vespalib::string> params; - if (external_expression) { - params.push_back("my_expr"); - index_env.addRankingExpression("my_expr", expression); - } else { - index_env.getProperties().add("self.rankingScript", expression); - } - Blueprint &bp = rank; - setup_ok = bp.setup(index_env, params); + setup_ok = rank.setup(index_env, {}); EXPECT_TRUE(!deps.accept_type_mismatch); } SetupResult::~SetupResult() = default; void verify_output_type(const TypeMap &object_inputs, - const vespalib::string &expression, const FeatureType &expect, - bool external_expression = false) + const vespalib::string &expression, const FeatureType &expect) { - SetupResult result(object_inputs, expression, external_expression); + SetupResult result(object_inputs, expression); EXPECT_TRUE(result.setup_ok); EXPECT_EQUAL(1u, result.deps.output.size()); ASSERT_EQUAL(1u, result.deps.output_type.size()); @@ -136,13 +126,6 @@ TEST("require that ranking expression can resolve to concrete complex type") { FeatureType::object(ValueType::from_spec("tensor(x{},y{},z{})")))); } -TEST("require that ranking expression can be external") { - TEST_DO(verify_output_type({}, "a*b", FeatureType::number(), true)); - TEST_DO(verify_output_type({{"b", "double"}}, "a*b", FeatureType::object(ValueType::double_type()), true)); - TEST_DO(verify_output_type({{"a", "tensor(x{},y{})"}, {"b", "tensor(y{},z{})"}}, "a*b", - FeatureType::object(ValueType::from_spec("tensor(x{},y{},z{})")), true)); -} - TEST("require that setup fails for incompatible types") { TEST_DO(verify_setup_fail({{"a", "tensor(x{},y{})"}, {"b", "tensor(y[10],z{})"}}, "a*b")); } diff --git a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp index 070d70997e6..8cba6013627 100644 --- a/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp +++ b/searchlib/src/vespa/searchlib/features/rankingexpressionfeature.cpp @@ -250,10 +250,7 @@ RankingExpressionBlueprint::setup(const fef::IIndexEnvironment &env, } //LOG(debug, "Script from config: '%s'\n", script.c_str()); } else if (params.size() == 1) { - script = env.getRankingExpression(params[0].getValue()); - if (script.empty()) { - script = params[0].getValue(); - } + script = params[0].getValue(); //LOG(debug, "Script from param: '%s'\n", script.c_str()); } else { return fail("No expression given."); diff --git a/searchlib/src/vespa/searchlib/fef/iindexenvironment.h b/searchlib/src/vespa/searchlib/fef/iindexenvironment.h index a01d7d8762b..0cd017d8ffa 100644 --- a/searchlib/src/vespa/searchlib/fef/iindexenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/iindexenvironment.h @@ -123,11 +123,6 @@ public: virtual std::unique_ptr<vespalib::eval::ConstantValue> getConstantValue(const vespalib::string &name) const = 0; /** - * Returns the ranking expression with the given name or empty string if not found. - **/ - virtual vespalib::string getRankingExpression(const vespalib::string &name) const = 0; - - /** * Get configuration for the given onnx model. **/ virtual const OnnxModel *getOnnxModel(const vespalib::string &name) const = 0; diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp index 85b20710f7a..d2d336dcdc8 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.cpp @@ -54,22 +54,6 @@ IndexEnvironment::addConstantValue(const vespalib::string &name, (void) insertRes; } -vespalib::string -IndexEnvironment::getRankingExpression(const vespalib::string &name) const -{ - auto pos = _expressions.find(name); - if (pos != _expressions.end()) { - return pos->second; - } - return {}; -} - -void -IndexEnvironment::addRankingExpression(const vespalib::string &name, const vespalib::string &value) -{ - _expressions.insert_or_assign(name, value); -} - const OnnxModel * IndexEnvironment::getOnnxModel(const vespalib::string &name) const { diff --git a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h index ca63117971a..0d8d0091921 100644 --- a/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h +++ b/searchlib/src/vespa/searchlib/fef/test/indexenvironment.h @@ -48,7 +48,6 @@ public: }; using ConstantsMap = std::map<vespalib::string, Constant>; - using ExprMap = std::map<vespalib::string, vespalib::string>; using ModelMap = std::map<vespalib::string, OnnxModel>; IndexEnvironment(); @@ -86,9 +85,6 @@ public: vespalib::eval::ValueType type, std::unique_ptr<vespalib::eval::Value> value); - vespalib::string getRankingExpression(const vespalib::string &name) const override; - void addRankingExpression(const vespalib::string &name, const vespalib::string &value); - const OnnxModel *getOnnxModel(const vespalib::string &name) const override; void addOnnxModel(const OnnxModel &model); @@ -102,7 +98,6 @@ private: AttributeMap _attrMap; TableManager _tableMan; ConstantsMap _constants; - ExprMap _expressions; ModelMap _models; }; diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index 7bc706f21ce..dc7be36c290 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -73,10 +73,6 @@ public: return vespalib::eval::ConstantValue::UP(); } - vespalib::string getRankingExpression(const vespalib::string &) const override { - return {}; - } - const search::fef::OnnxModel *getOnnxModel(const vespalib::string &) const override { return nullptr; } |