From a9d670dff75d1f1d3b9849ee60e312717ca570fe Mon Sep 17 00:00:00 2001 From: Tor Egge Date: Wed, 7 Sep 2022 15:08:55 +0200 Subject: Revert "Unify access to assets needed during rank-setup." --- .../src/apps/verify_ranksetup/verify_ranksetup.cpp | 44 ++++++++------------- .../documentdb/configurer/configurer_test.cpp | 2 +- .../documentdbconfig/documentdbconfig_test.cpp | 11 +++--- .../fileconfigmanager/fileconfigmanager_test.cpp | 10 ++--- .../constant_value_repo_test.cpp | 24 +++++------ .../index_environment/index_environment_test.cpp | 42 ++++++-------------- .../src/tests/proton/matching/matching_test.cpp | 16 ++------ .../verify_ranksetup/verify_ranksetup_test.cpp | 28 ++++++------- .../searchcore/proton/bucketdb/bucketstate.cpp | 14 +++++-- .../searchcore/proton/matching/CMakeLists.txt | 2 +- .../proton/matching/constant_value_repo.cpp | 31 +++++++++++++++ .../proton/matching/constant_value_repo.h | 30 ++++++++++++++ .../proton/matching/i_constant_value_repo.h | 18 +++++++++ .../proton/matching/i_ranking_assets_repo.h | 21 ---------- .../proton/matching/indexenvironment.cpp | 24 +++++++++-- .../searchcore/proton/matching/indexenvironment.h | 27 +++++++------ .../vespa/searchcore/proton/matching/matcher.cpp | 18 ++++++--- .../src/vespa/searchcore/proton/matching/matcher.h | 14 +++++-- .../searchcore/proton/matching/onnx_models.cpp | 15 ++++--- .../vespa/searchcore/proton/matching/onnx_models.h | 6 +-- .../proton/matching/ranking_assets_repo.cpp | 46 ---------------------- .../proton/matching/ranking_assets_repo.h | 37 ----------------- .../proton/matching/ranking_constants.cpp | 5 +-- .../searchcore/proton/matching/ranking_constants.h | 6 +-- .../proton/matching/ranking_expressions.cpp | 3 +- .../proton/matching/ranking_expressions.h | 8 ++-- .../vespa/searchcore/proton/server/buckethandler.h | 2 +- .../searchcore/proton/server/bucketmovejob.cpp | 12 +++--- .../vespa/searchcore/proton/server/bucketmovejob.h | 6 +-- .../vespa/searchcore/proton/server/documentdb.cpp | 6 +-- .../proton/server/documentdbconfigmanager.cpp | 4 +- .../proton/server/maintenance_jobs_injector.cpp | 11 +++--- .../vespa/searchcore/proton/server/matchers.cpp | 14 ++++++- .../src/vespa/searchcore/proton/server/matchers.h | 9 +++-- .../proton/server/sample_attribute_usage_job.cpp | 4 +- .../server/searchable_doc_subdb_configurer.cpp | 27 +++++++------ .../server/searchable_doc_subdb_configurer.h | 12 +++--- .../proton/server/searchabledocsubdb.cpp | 11 +++--- .../searchcore/proton/server/searchabledocsubdb.h | 4 +- 39 files changed, 302 insertions(+), 322 deletions(-) create mode 100644 searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp create mode 100644 searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.h create mode 100644 searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h delete mode 100644 searchcore/src/vespa/searchcore/proton/matching/i_ranking_assets_repo.h delete mode 100644 searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.cpp delete mode 100644 searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.h (limited to 'searchcore') diff --git a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp index 43d0b709a1b..dcfbc34653d 100644 --- a/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp +++ b/searchcore/src/apps/verify_ranksetup/verify_ranksetup.cpp @@ -18,7 +18,6 @@ #include #include #include -#include #include #include #include @@ -33,7 +32,7 @@ using config::ConfigRuntimeException; using config::ConfigSubscriber; using config::IConfigContext; using config::InvalidConfigException; -using proton::matching::IRankingAssetsRepo; +using proton::matching::IConstantValueRepo; using proton::matching::RankingExpressions; using proton::matching::OnnxModels; using vespa::config::search::AttributesConfig; @@ -92,7 +91,7 @@ make_models(const OnnxModelsConfig &modelsCfg, const VerifyRanksetupConfig &myCf entry.name.c_str(), entry.fileref.c_str())); } } - return OnnxModels(std::move(model_list)); + return OnnxModels(model_list); } class VerifyRankSetup @@ -101,7 +100,9 @@ private: std::vector _messages; bool verify(const search::index::Schema &schema, const search::fef::Properties &props, - const IRankingAssetsRepo &repo); + const IConstantValueRepo &repo, + const RankingExpressions &expressions, + const OnnxModels &models); bool verifyConfig(const VerifyRanksetupConfig &myCfg, const RankProfilesConfig &rankCfg, @@ -118,28 +119,14 @@ public: bool verify(const std::string & configId); }; -struct DummyRankingAssetsRepo : IRankingAssetsRepo { +struct DummyConstantValueRepo : IConstantValueRepo { const RankingConstantsConfig &cfg; - RankingExpressions _expressions; - OnnxModels _onnxModels; - DummyRankingAssetsRepo(const RankingConstantsConfig &cfg_in, RankingExpressions expressions, OnnxModels onnxModels) - : cfg(cfg_in), - _expressions(std::move(expressions)), - _onnxModels(std::move(onnxModels)) - {} + DummyConstantValueRepo(const RankingConstantsConfig &cfg_in) : cfg(cfg_in) {} vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override; - - vespalib::string getExpression(const vespalib::string & name) const override { - return _expressions.loadExpression(name); - } - - const search::fef::OnnxModel *getOnnxModel(const vespalib::string & name) const override { - return _onnxModels.getModel(name); - } }; vespalib::eval::ConstantValue::UP -DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const { +DummyConstantValueRepo::getConstant(const vespalib::string &name) const { for (const auto &entry: cfg.constant) { if (entry.name == name) { try { @@ -150,7 +137,7 @@ DummyRankingAssetsRepo::getConstant(const vespalib::string &name) const { } } } - return {}; + return vespalib::eval::ConstantValue::UP(nullptr); } VerifyRankSetup::VerifyRankSetup() @@ -162,9 +149,11 @@ VerifyRankSetup::~VerifyRankSetup() = default; bool VerifyRankSetup::verify(const search::index::Schema &schema, const search::fef::Properties &props, - const IRankingAssetsRepo &repo) + const IConstantValueRepo &repo, + const RankingExpressions &expressions, + const OnnxModels &models) { - proton::matching::IndexEnvironment indexEnv(0, schema, props, repo); + proton::matching::IndexEnvironment indexEnv(0, schema, props, repo, expressions, models); search::fef::BlueprintFactory factory; search::features::setup_search_features(factory); search::fef::test::setup_fef_test_plugin(factory); @@ -203,8 +192,9 @@ VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, search::index::Schema schema; search::index::SchemaBuilder::build(schemaCfg, schema); search::index::SchemaBuilder::build(attributeCfg, schema); - DummyRankingAssetsRepo repo(constantsCfg, make_expressions(expressionsCfg, myCfg, _messages), - make_models(modelsCfg, myCfg, _messages)); + DummyConstantValueRepo repo(constantsCfg); + auto expressions = make_expressions(expressionsCfg, myCfg, _messages); + auto models = make_models(modelsCfg, myCfg, _messages); for(size_t i = 0; i < rankCfg.rankprofile.size(); i++) { search::fef::Properties properties; const RankProfilesConfig::Rankprofile &profile = rankCfg.rankprofile[i]; @@ -212,7 +202,7 @@ VerifyRankSetup::verifyConfig(const VerifyRanksetupConfig &myCfg, properties.add(profile.fef.property[j].name, profile.fef.property[j].value); } - if (verify(schema, properties, repo)) { + if (verify(schema, properties, repo, expressions, models)) { _messages.emplace_back(search::fef::Level::INFO, fmt("rank profile '%s': pass", profile.name.c_str())); } else { diff --git a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp index 53ea1b3542f..8bd1168d9af 100644 --- a/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp +++ b/searchcore/src/tests/proton/documentdb/configurer/configurer_test.cpp @@ -152,7 +152,7 @@ struct Fixture vespalib::TestClock _clock; matching::QueryLimiter _queryLimiter; EmptyConstantValueFactory _constantValueFactory; - RankingAssetsRepo _constantValueRepo; + ConstantValueRepo _constantValueRepo; vespalib::ThreadStackExecutor _summaryExecutor; std::shared_ptr _pendingLidsForCommit; ViewSet _views; diff --git a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp index c2d3da1b4f6..c3b117f300a 100644 --- a/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp +++ b/searchcore/src/tests/proton/documentdb/documentdbconfig/documentdbconfig_test.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -71,13 +72,13 @@ public: return *this; } MyConfigBuilder &addRankingExpression() { - _builder.rankingExpressions(make_shared(std::move(RankingExpressions().add("my_expr", "my_file")))); + auto expr_list = RankingExpressions().add("my_expr", "my_file"); + _builder.rankingExpressions(make_shared(expr_list)); return *this; } MyConfigBuilder &addOnnxModel() { - OnnxModels::Vector models; - models.emplace_back("my_model_name", "my_model_file"); - _builder.onnxModels(make_shared(std::move(models))); + OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; + _builder.onnxModels(make_shared(models)); return *this; } MyConfigBuilder &addImportedField() { @@ -168,7 +169,7 @@ struct DelayAttributeAspectFixture { Schema::SP schema; ConfigSP attrCfg; ConfigSP noAttrCfg; - explicit DelayAttributeAspectFixture(bool hasDocField) + DelayAttributeAspectFixture(bool hasDocField) : schema(make_shared()), attrCfg(), noAttrCfg() diff --git a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp index e1ec5916291..ce66e99d2fc 100644 --- a/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp +++ b/searchcore/src/tests/proton/documentdb/fileconfigmanager/fileconfigmanager_test.cpp @@ -113,13 +113,11 @@ addConfigsThatAreNotSavedToDisk(const DocumentDBConfig &cfg) RankingConstants::Vector constants = {{"my_name", "my_type", "my_path"}}; builder.rankingConstants(std::make_shared(constants)); - auto expr_list = std::make_shared(); - expr_list->add("my_expr", "my_file"); - builder.rankingExpressions(expr_list); + auto expr_list = RankingExpressions().add("my_expr", "my_file"); + builder.rankingExpressions(std::make_shared(expr_list)); - OnnxModels::Vector models; - models.emplace_back("my_model_name", "my_model_file"); - builder.onnxModels(std::make_shared(std::move(models))); + OnnxModels::Vector models = {{"my_model_name", "my_model_file"}}; + builder.onnxModels(std::make_shared(models)); ImportedFieldsConfigBuilder importedFields; importedFields.attribute.resize(1); diff --git a/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp b/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp index 3c157221e60..4c06feb6ae7 100644 --- a/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp +++ b/searchcore/src/tests/proton/matching/constant_value_repo/constant_value_repo_test.cpp @@ -2,7 +2,7 @@ #include -#include +#include #include using namespace proton::matching; @@ -14,9 +14,9 @@ private: ValueType _type; public: - explicit DoubleConstantValue(double value_) : _value(value_), _type(ValueType::double_type()) {} - const ValueType &type() const override { return _type; } - const Value &value() const override { return _value; } + DoubleConstantValue(double value_) : _value(value_), _type(ValueType::double_type()) {} + virtual const ValueType &type() const override { return _type; } + virtual const Value &value() const override { return _value; } }; class MyConstantValueFactory : public ConstantValueFactory { @@ -30,7 +30,7 @@ public: void add(const vespalib::string &path, const vespalib::string &type, double value) { _map.insert(std::make_pair(std::make_pair(path, type), value)); } - ConstantValue::UP create(const vespalib::string &path, const vespalib::string &type) const override { + virtual ConstantValue::UP create(const vespalib::string &path, const vespalib::string &type) const override { auto itr = _map.find(std::make_pair(path, type)); if (itr != _map.end()) { return std::make_unique(itr->second); @@ -41,16 +41,14 @@ public: struct Fixture { MyConstantValueFactory factory; - RankingAssetsRepo repo; + ConstantValueRepo repo; Fixture() : factory(), repo(factory) { factory.add("path_1", "double", 3); factory.add("path_2", "double", 5); - RankingConstants::Vector constants; - constants.emplace_back("foo", "double", "path_1"); - constants.emplace_back("bar", "double", "path_3"); - repo.reconfigure(std::make_shared(constants),{}, {}); + repo.reconfigure(RankingConstants({{"foo", "double", "path_1"}, + {"bar", "double", "path_3"}})); } }; @@ -71,10 +69,8 @@ TEST_F("require that non-existing constant value in factory returns bad constant TEST_F("require that reconfigure replaces existing constant values in repo", Fixture) { - RankingConstants::Vector constants; - constants.emplace_back("bar", "double", "path_3"); - constants.emplace_back("baz", "double", "path_2"); - f.repo.reconfigure(std::make_shared(constants), {}, {}); + f.repo.reconfigure(RankingConstants({{"bar", "double", "path_3"}, + {"baz", "double", "path_2"}})); f.factory.add("path_3", "double", 7); EXPECT_TRUE(f.repo.getConstant("foo").get() == nullptr); EXPECT_EQUAL(7, f.repo.getConstant("bar")->value().as_double()); diff --git a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp index bd7c3a4e8fd..f6c10f7ddb5 100644 --- a/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp +++ b/searchcore/src/tests/proton/matching/index_environment/index_environment_test.cpp @@ -4,7 +4,6 @@ #include #include #include -#include using namespace proton::matching; using search::fef::FieldInfo; @@ -34,34 +33,17 @@ RankingExpressions make_expressions() { OnnxModels make_models() { OnnxModels::Vector list; - list.emplace_back(std::move(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1"))); + list.emplace_back(OnnxModel("model1", "path1").input_feature("input1","feature1").output_name("output1", "out1")); list.emplace_back(OnnxModel("model2", "path2")); - return {std::move(list)}; + return OnnxModels(list); } -struct MyRankingAssetsRepo : public IRankingAssetsRepo { - RankingExpressions _expressions; - OnnxModels _onnxModels; - MyRankingAssetsRepo(RankingExpressions expressions, OnnxModels onnxModels) - : _expressions(std::move(expressions)), - _onnxModels(std::move(onnxModels)) - {} - ~MyRankingAssetsRepo() override; - ConstantValue::UP getConstant(const vespalib::string &) const override { - return {}; - } - - vespalib::string getExpression(const vespalib::string & name) const override { - return _expressions.loadExpression(name); - } - - const OnnxModel *getOnnxModel(const vespalib::string & name) const override { - return _onnxModels.getModel(name); +struct MyConstantValueRepo : public IConstantValueRepo { + virtual ConstantValue::UP getConstant(const vespalib::string &) const override { + return ConstantValue::UP(); } }; -MyRankingAssetsRepo::~MyRankingAssetsRepo() = default; - Schema::UP buildSchema() { @@ -78,19 +60,19 @@ buildEmptySchema() } struct Fixture { - MyRankingAssetsRepo repo; + MyConstantValueRepo repo; Schema::UP schema; IndexEnvironment env; - explicit Fixture(Schema::UP schema_) - : repo(make_expressions(), make_models()), + Fixture(Schema::UP schema_) + : repo(), schema(std::move(schema_)), - env(7, *schema, Properties(), repo) + env(7, *schema, Properties(), repo, make_expressions(), make_models()) { } const FieldInfo *assertField(size_t idx, const vespalib::string &name, DataType dataType, - CollectionType collectionType) const { + CollectionType collectionType) { const FieldInfo *field = env.getField(idx); ASSERT_TRUE(field != nullptr); EXPECT_EQUAL(field, env.getFieldByName(name)); @@ -103,7 +85,7 @@ struct Fixture { void assertHiddenAttributeField(size_t idx, const vespalib::string &name, DataType dataType, - CollectionType collectionType) const { + CollectionType collectionType) { const FieldInfo *field = assertField(idx, name, dataType, collectionType); EXPECT_FALSE(field->hasAttribute()); EXPECT_TRUE(field->type() == FieldType::HIDDEN_ATTRIBUTE); @@ -112,7 +94,7 @@ struct Fixture { void assertAttributeField(size_t idx, const vespalib::string &name, DataType dataType, - CollectionType collectionType) const { + CollectionType collectionType) { const FieldInfo *field = assertField(idx, name, dataType, collectionType); EXPECT_TRUE(field->hasAttribute()); EXPECT_TRUE(field->type() == FieldType::ATTRIBUTE); diff --git a/searchcore/src/tests/proton/matching/matching_test.cpp b/searchcore/src/tests/proton/matching/matching_test.cpp index b94ef025aa6..dbf4200a24a 100644 --- a/searchcore/src/tests/proton/matching/matching_test.cpp +++ b/searchcore/src/tests/proton/matching/matching_test.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include #include #include #include @@ -107,18 +107,10 @@ vespalib::string make_same_element_stack_dump(const vespalib::string &a1_term, c const uint32_t NUM_DOCS = 1000; -struct EmptyRankingAssetsRepo : public proton::matching::IRankingAssetsRepo { +struct EmptyConstantValueRepo : public proton::matching::IConstantValueRepo { vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &) const override { return {}; } - - vespalib::string getExpression(const vespalib::string &) const override { - return {}; - } - - const OnnxModel *getOnnxModel(const vespalib::string &) const override { - return nullptr; - } }; //----------------------------------------------------------------------------- @@ -133,7 +125,7 @@ struct MyWorld { MatchingStats matchingStats; vespalib::TestClock clock; QueryLimiter queryLimiter; - EmptyRankingAssetsRepo constantValueRepo; + EmptyConstantValueRepo constantValueRepo; MyWorld(); ~MyWorld(); @@ -355,7 +347,7 @@ struct MyWorld { } Matcher::SP createMatcher() { - return std::make_shared(schema, config, clock.clock(), queryLimiter, constantValueRepo, 0); + return std::make_shared(schema, config, clock.clock(), queryLimiter, constantValueRepo, RankingExpressions(), OnnxModels(), 0); } struct MySearchHandler : ISearchHandler { diff --git a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp index 33a52f70f2b..6f4420695c4 100644 --- a/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp +++ b/searchcore/src/tests/proton/verify_ranksetup/verify_ranksetup_test.cpp @@ -27,9 +27,9 @@ using vespalib::make_string_short::fmt; struct Writer { FILE *file; - explicit Writer(const std::string &file_name) { + Writer(const std::string &file_name) { file = fopen(file_name.c_str(), "w"); - ASSERT_TRUE(file != nullptr); + ASSERT_TRUE(file != 0); } void fmt(const char *format, ...) const #ifdef __GNUC__ @@ -63,7 +63,7 @@ struct Attribute { ~Attribute(); }; -Attribute::~Attribute() = default; +Attribute::~Attribute() {} struct Setup { std::map > indexes; @@ -75,8 +75,8 @@ struct Setup { std::map onnx_models; Setup(); ~Setup(); - void add_onnx_model(OnnxModel model) { - onnx_models.insert_or_assign(model.name(), std::move(model)); + void add_onnx_model(const OnnxModel &model) { + onnx_models.insert_or_assign(model.name(), model); } void index(const std::string &name, schema::DataType data_type, schema::CollectionType collection_type) @@ -141,7 +141,7 @@ struct Setup { } void write_indexschema(const Writer &out) { out.fmt("indexfield[%zu]\n", indexes.size()); - auto pos = indexes.begin(); + std::map >::const_iterator pos = indexes.begin(); for (size_t i = 0; pos != indexes.end(); ++pos, ++i) { out.fmt("indexfield[%zu].name \"%s\"\n", i, pos->first.c_str()); out.fmt("indexfield[%zu].datatype %s\n", i, pos->second.first.c_str()); @@ -151,7 +151,7 @@ struct Setup { void write_rank_profiles(const Writer &out) { out.fmt("rankprofile[%zu]\n", extra_profiles.size() + 1); out.fmt("rankprofile[0].name \"default\"\n"); - auto pos = properties.begin(); + std::map::const_iterator pos = properties.begin(); for (size_t i = 0; pos != properties.end(); ++pos, ++i) { out.fmt("rankprofile[0].fef.property[%zu]\n", properties.size()); out.fmt("rankprofile[0].fef.property[%zu].name \"%s\"\n", i, pos->first.c_str()); @@ -258,7 +258,7 @@ Setup::Setup() { verify_dir(); } -Setup::~Setup() = default; +Setup::~Setup() {} //----------------------------------------------------------------------------- @@ -279,15 +279,15 @@ struct SimpleSetup : Setup { struct OnnxSetup : Setup { OnnxSetup() : Setup() { add_onnx_model(OnnxModel("simple", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx"))); - add_onnx_model(std::move(OnnxModel("mapped", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx")) + add_onnx_model(OnnxModel("mapped", TEST_PATH("../../../../../eval/src/tests/tensor/onnx_wrapper/simple.onnx")) .input_feature("query_tensor", "rankingExpression(qt)") .input_feature("attribute_tensor", "rankingExpression(at)") .input_feature("bias_tensor", "rankingExpression(bt)") - .output_name("output", "result"))); - add_onnx_model(std::move(OnnxModel("fragile", TEST_PATH("../../../../../searchlib/src/tests/features/onnx_feature/fragile.onnx")) - .dry_run_on_setup(true))); - add_onnx_model(std::move(OnnxModel("unfragile", TEST_PATH("../../../../../searchlib/src/tests/features/onnx_feature/fragile.onnx")) - .dry_run_on_setup(false))); + .output_name("output", "result")); + add_onnx_model(OnnxModel("fragile", TEST_PATH("../../../../../searchlib/src/tests/features/onnx_feature/fragile.onnx")) + .dry_run_on_setup(true)); + add_onnx_model(OnnxModel("unfragile", TEST_PATH("../../../../../searchlib/src/tests/features/onnx_feature/fragile.onnx")) + .dry_run_on_setup(false)); } }; diff --git a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp index 4c25657ab43..e041f9991fe 100644 --- a/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp +++ b/searchcore/src/vespa/searchcore/proton/bucketdb/bucketstate.cpp @@ -146,6 +146,8 @@ BucketState::operator+=(const BucketState &rhs) { for (uint32_t i = 0; i < COUNTS; ++i) { _docCount[i] += rhs._docCount[i]; + } + for (uint32_t i = 0; i < COUNTS; ++i) { _docSizes[i] += rhs._docSizes[i]; } @@ -165,10 +167,14 @@ BucketState::operator-=(const BucketState &rhs) { for (uint32_t i = 0; i < COUNTS; ++i) { assert(_docCount[i] >= rhs._docCount[i]); + } + for (uint32_t i = 0; i < COUNTS; ++i) { assert(_docSizes[i] >= rhs._docSizes[i]); } for (uint32_t i = 0; i < COUNTS; ++i) { _docCount[i] -= rhs._docCount[i]; + } + for (uint32_t i = 0; i < COUNTS; ++i) { _docSizes[i] -= rhs._docSizes[i]; } switch (_checksumType) { @@ -204,9 +210,11 @@ BucketState::operator storage::spi::BucketInfo() const using BucketInfo = storage::spi::BucketInfo; - return {getChecksum(), documentCount, uint32_t(docSizes), entryCount, uint32_t(entrySizes), - notReady > 0 ? BucketInfo::NOT_READY : BucketInfo::READY, - _active ? BucketInfo::ACTIVE : BucketInfo::NOT_ACTIVE}; + return BucketInfo(getChecksum(), + documentCount, docSizes, + entryCount, entrySizes, + notReady > 0 ? BucketInfo::NOT_READY : BucketInfo::READY, + _active ? BucketInfo::ACTIVE : BucketInfo::NOT_ACTIVE); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt index 597312843c9..1c203dd1284 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt +++ b/searchcore/src/vespa/searchcore/proton/matching/CMakeLists.txt @@ -3,7 +3,7 @@ vespa_add_library(searchcore_matching STATIC SOURCES attribute_limiter.cpp blueprintbuilder.cpp - ranking_assets_repo.cpp + constant_value_repo.cpp docid_range_scheduler.cpp docsum_matcher.cpp document_scorer.cpp diff --git a/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp b/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp new file mode 100644 index 00000000000..05448d2c83b --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.cpp @@ -0,0 +1,31 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#include "constant_value_repo.h" + +using vespalib::eval::ConstantValue; + +namespace proton::matching { + +ConstantValueRepo::ConstantValueRepo(const ConstantValueFactory &factory) + : _factory(factory), + _constants() +{ +} + +void +ConstantValueRepo::reconfigure(const RankingConstants &constants) +{ + _constants = constants; +} + +ConstantValue::UP +ConstantValueRepo::getConstant(const vespalib::string &name) const +{ + const RankingConstants::Constant *constant = _constants.getConstant(name); + if (constant != nullptr) { + return _factory.create(constant->filePath, constant->type); + } + return ConstantValue::UP(nullptr); +} + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.h b/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.h new file mode 100644 index 00000000000..6bf91616019 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/constant_value_repo.h @@ -0,0 +1,30 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "i_constant_value_repo.h" +#include "ranking_constants.h" +#include + +namespace proton::matching { + +/** + * Class that provides access to a configured set of rank constant values. + * + * This class maps the symbolic name used by rank features to the file path (where the constant is stored) and its type, + * and uses a factory to instantiate the actual constant values. + */ +class ConstantValueRepo : public IConstantValueRepo { +private: + using ConstantValueFactory = vespalib::eval::ConstantValueFactory; + + const ConstantValueFactory &_factory; + RankingConstants _constants; + +public: + ConstantValueRepo(const ConstantValueFactory &factory); + void reconfigure(const RankingConstants &constants); + virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override; +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h b/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h new file mode 100644 index 00000000000..3bc2dc642d3 --- /dev/null +++ b/searchcore/src/vespa/searchcore/proton/matching/i_constant_value_repo.h @@ -0,0 +1,18 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include + +namespace proton::matching { + +/** + * Interface for retrieving a named constant rank value to be used by features in the rank framework. + * If the given value is not found a nullptr should be returned. + */ +struct IConstantValueRepo { + virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const = 0; + virtual ~IConstantValueRepo() {} +}; + +} diff --git a/searchcore/src/vespa/searchcore/proton/matching/i_ranking_assets_repo.h b/searchcore/src/vespa/searchcore/proton/matching/i_ranking_assets_repo.h deleted file mode 100644 index d96423bf25c..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matching/i_ranking_assets_repo.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include - -namespace search::fef { class OnnxModel; } -namespace proton::matching { - -/** - * Interface for retrieving named constants, expressions and models from ranking. - * Empty strings or nullptrs indicates nothing found. - */ -struct IRankingAssetsRepo { - virtual vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const = 0; - virtual vespalib::string getExpression(const vespalib::string &name) const = 0; - virtual const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const = 0; - virtual ~IRankingAssetsRepo() = default; -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp index a90dfea8f40..f13137d182c 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.cpp @@ -63,14 +63,18 @@ IndexEnvironment::insertField(const search::fef::FieldInfo &field) IndexEnvironment::IndexEnvironment(uint32_t distributionKey, const search::index::Schema &schema, - search::fef::Properties props, - const IRankingAssetsRepo &rankingAssetsRepo) + const search::fef::Properties &props, + const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, + OnnxModels onnxModels) : _tableManager(), - _properties(std::move(props)), + _properties(props), _fieldNames(), _fields(), _motivation(UNKNOWN), - _rankingAssetsRepo(rankingAssetsRepo), + _constantValueRepo(constantValueRepo), + _rankingExpressions(std::move(rankingExpressions)), + _onnxModels(std::move(onnxModels)), _distributionKey(distributionKey) { _tableManager.addFactory(std::make_shared(256)); @@ -129,6 +133,18 @@ IndexEnvironment::hintFieldAccess(uint32_t ) const { } void IndexEnvironment::hintAttributeAccess(const string &) const { } +vespalib::string +IndexEnvironment::getRankingExpression(const vespalib::string &name) const +{ + return _rankingExpressions.loadExpression(name); +} + +const search::fef::OnnxModel * +IndexEnvironment::getOnnxModel(const vespalib::string &name) const +{ + return _onnxModels.getModel(name); +} + IndexEnvironment::~IndexEnvironment() = default; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h index 8f8ac516ab8..ee1b5f2e938 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h +++ b/searchcore/src/vespa/searchcore/proton/matching/indexenvironment.h @@ -2,7 +2,9 @@ #pragma once -#include "i_ranking_assets_repo.h" +#include "onnx_models.h" +#include "ranking_expressions.h" +#include "i_constant_value_repo.h" #include #include #include @@ -24,7 +26,9 @@ private: FieldNameMap _fieldNames; std::vector _fields; mutable FeatureMotivation _motivation; - const IRankingAssetsRepo &_rankingAssetsRepo; + const IConstantValueRepo &_constantValueRepo; + RankingExpressions _rankingExpressions; + OnnxModels _onnxModels; uint32_t _distributionKey; @@ -49,10 +53,10 @@ public: **/ IndexEnvironment(uint32_t distributionKey, const search::index::Schema &schema, - search::fef::Properties props, - const IRankingAssetsRepo &constantValueRepo); - ~IndexEnvironment() override; - + const search::fef::Properties &props, + const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, + OnnxModels onnxModels); const search::fef::Properties &getProperties() const override; uint32_t getNumFields() const override; @@ -66,15 +70,12 @@ public: uint32_t getDistributionKey() const override { return _distributionKey; } vespalib::eval::ConstantValue::UP getConstantValue(const vespalib::string &name) const override { - return _rankingAssetsRepo.getConstant(name); - } - vespalib::string getRankingExpression(const vespalib::string &name) const override { - return _rankingAssetsRepo.getExpression(name); + return _constantValueRepo.getConstant(name); } + vespalib::string getRankingExpression(const vespalib::string &name) const override; - const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override { - return _rankingAssetsRepo.getOnnxModel(name); - } + const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override; + ~IndexEnvironment() override; }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp index 6dc1b2b354e..108397e6e5a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.cpp @@ -50,14 +50,14 @@ struct StupidMetaStore : search::IDocumentMetaStore { bool getGid(DocId, GlobalId &) const override { return false; } bool getGidEvenIfMoved(DocId, GlobalId &) const override { return false; } bool getLid(const GlobalId &, DocId &) const override { return false; } - DocumentMetaData getMetaData(const GlobalId &) const override { return {}; } + DocumentMetaData getMetaData(const GlobalId &) const override { return DocumentMetaData(); } void getMetaData(const BucketId &, DocumentMetaData::Vector &) const override { } DocId getCommittedDocIdLimit() const override { return 1; } DocId getNumUsedLids() const override { return 0; } DocId getNumActiveLids() const override { return 0; } uint64_t getCurrentGeneration() const override { return 0; } - LidUsageStats getLidUsageStats() const override { return {}; } - Blueprint::UP createWhiteListBlueprint() const override { return {}; } + LidUsageStats getLidUsageStats() const override { return LidUsageStats(); } + Blueprint::UP createWhiteListBlueprint() const override { return Blueprint::UP(); } void foreach(const search::IGidToLidMapperVisitor &) const override { } }; @@ -100,9 +100,10 @@ handleGroupingSession(SessionManager &sessionMgr, GroupingContext & groupingCont } // namespace proton::matching:: -Matcher::Matcher(const search::index::Schema &schema, Properties props, const vespalib::Clock &clock, - QueryLimiter &queryLimiter, const IRankingAssetsRepo &rankingAssetsRepo, uint32_t distributionKey) - : _indexEnv(distributionKey, schema, std::move(props), rankingAssetsRepo), +Matcher::Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, + QueryLimiter &queryLimiter, const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, OnnxModels onnxModels, uint32_t distributionKey) + : _indexEnv(distributionKey, schema, props, constantValueRepo, std::move(rankingExpressions), std::move(onnxModels)), _blueprintFactory(), _rankSetup(), _viewResolver(ViewResolver::createFromSchema(schema)), @@ -385,4 +386,9 @@ Matcher::canProduceSummaryFeatures() const { return ! _rankSetup->getSummaryFeatures().empty(); } +double +Matcher::get_termwise_limit() const { + return _rankSetup->get_termwise_limit(); +} + } diff --git a/searchcore/src/vespa/searchcore/proton/matching/matcher.h b/searchcore/src/vespa/searchcore/proton/matching/matcher.h index ef79818a0b3..4ef57d69d47 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/matcher.h +++ b/searchcore/src/vespa/searchcore/proton/matching/matcher.h @@ -2,7 +2,7 @@ #pragma once -#include "i_ranking_assets_repo.h" +#include "i_constant_value_repo.h" #include "indexenvironment.h" #include "matching_stats.h" #include "search_session.h" @@ -69,7 +69,10 @@ private: size_t computeNumThreadsPerSearch(search::queryeval::Blueprint::HitEstimate hits, const Properties & rankProperties) const; public: - using SP = std::shared_ptr; + /** + * Convenience typedefs. + */ + typedef std::shared_ptr SP; Matcher(const Matcher &) = delete; @@ -84,9 +87,11 @@ public: * @param props ranking configuration * @param clock used for timeout handling **/ - Matcher(const search::index::Schema &schema, Properties props, + Matcher(const search::index::Schema &schema, const Properties &props, const vespalib::Clock &clock, QueryLimiter &queryLimiter, - const IRankingAssetsRepo &rankingAssetsRepo, uint32_t distributionKey); + const IConstantValueRepo &constantValueRepo, + RankingExpressions rankingExpressions, OnnxModels onnxModels, + uint32_t distributionKey); const search::fef::IIndexEnvironment &get_index_env() const { return _indexEnv; } @@ -174,6 +179,7 @@ public: * @return true if this rankprofile has summary-features enabled **/ bool canProduceSummaryFeatures() const; + double get_termwise_limit() const; }; } diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp index 399e4ab0ad0..01cdd904389 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.cpp @@ -1,19 +1,22 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "onnx_models.h" -#include +#include namespace proton::matching { -OnnxModels::OnnxModels() = default; -OnnxModels::OnnxModels(OnnxModels &&) noexcept = default; +OnnxModels::OnnxModels() + : _models() +{ +} + OnnxModels::~OnnxModels() = default; -OnnxModels::OnnxModels(Vector models) +OnnxModels::OnnxModels(const Vector &models) : _models() { - for (auto &model: models) { - _models.emplace(model.name(), std::move(model)); + for (const auto &model: models) { + _models.emplace(model.name(), model); } } diff --git a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h index c0582600973..4907c42128b 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h +++ b/searchcore/src/vespa/searchcore/proton/matching/onnx_models.h @@ -27,11 +27,7 @@ private: public: using SP = std::shared_ptr; OnnxModels(); - OnnxModels(Vector models); - OnnxModels(OnnxModels &&) noexcept; - OnnxModels & operator=(OnnxModels &&) = delete; - OnnxModels(const OnnxModels &) = delete; - OnnxModels & operator =(const OnnxModels &) = delete; + OnnxModels(const Vector &models); ~OnnxModels(); bool operator==(const OnnxModels &rhs) const; const Model *getModel(const vespalib::string &name) const; diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.cpp b/searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.cpp deleted file mode 100644 index 783a34389d8..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.cpp +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#include "ranking_assets_repo.h" - -using vespalib::eval::ConstantValue; - -namespace proton::matching { - -RankingAssetsRepo::RankingAssetsRepo(const ConstantValueFactory &factory) - : _factory(factory), - _constants() -{ -} - -RankingAssetsRepo::~RankingAssetsRepo() = default; - -void -RankingAssetsRepo::reconfigure(RankingConstants::SP constants, RankingExpressions::SP expressions, OnnxModels::SP models) -{ - _constants = std::move(constants); - _rankingExpressions = std::move(expressions); - _onnxModels = std::move(models); -} - -ConstantValue::UP -RankingAssetsRepo::getConstant(const vespalib::string &name) const -{ - if ( ! _constants) return {}; - const RankingConstants::Constant *constant = _constants->getConstant(name); - if (constant != nullptr) { - return _factory.create(constant->filePath, constant->type); - } - return {}; -} - -vespalib::string -RankingAssetsRepo::getExpression(const vespalib::string &name) const { - return _rankingExpressions ? _rankingExpressions->loadExpression(name) : ""; -} - -const search::fef::OnnxModel * -RankingAssetsRepo::getOnnxModel(const vespalib::string &name) const { - return _onnxModels ? _onnxModels->getModel(name) : nullptr; -} - -} diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.h b/searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.h deleted file mode 100644 index d0440f7be90..00000000000 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_assets_repo.h +++ /dev/null @@ -1,37 +0,0 @@ -// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. - -#pragma once - -#include "i_ranking_assets_repo.h" -#include "ranking_constants.h" -#include "onnx_models.h" -#include "ranking_expressions.h" -#include - -namespace proton::matching { - -/** - * Class that provides access to a configured set of rank constant values. - * - * This class maps symbolic names to assets used while setting up rank features blueprints. - * A factory is used to instantiate constant values. - */ -class RankingAssetsRepo : public IRankingAssetsRepo { -private: - using ConstantValueFactory = vespalib::eval::ConstantValueFactory; - - const ConstantValueFactory &_factory; - RankingConstants::SP _constants; - RankingExpressions::SP _rankingExpressions; - OnnxModels::SP _onnxModels; - -public: - explicit RankingAssetsRepo(const ConstantValueFactory &factory); - ~RankingAssetsRepo() override; - void reconfigure(RankingConstants::SP constants, RankingExpressions::SP expressions, OnnxModels::SP models); - vespalib::eval::ConstantValue::UP getConstant(const vespalib::string &name) const override; - vespalib::string getExpression(const vespalib::string &name) const override; - const search::fef::OnnxModel *getOnnxModel(const vespalib::string &name) const override; -}; - -} diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.cpp b/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.cpp index 02f705ba4ea..fbf3020611a 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.cpp @@ -13,7 +13,7 @@ RankingConstants::Constant::Constant(const vespalib::string &name_in, { } -RankingConstants::Constant::~Constant() = default; +RankingConstants::Constant::~Constant() {} bool RankingConstants::Constant::operator==(const Constant &rhs) const @@ -28,8 +28,7 @@ RankingConstants::RankingConstants() { } -RankingConstants::~RankingConstants() = default; -RankingConstants::RankingConstants(RankingConstants &&) noexcept = default; +RankingConstants::~RankingConstants() {} RankingConstants::RankingConstants(const Vector &constants) : _constants() diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.h b/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.h index c527c6e5571..bae04c66d1e 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.h +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_constants.h @@ -35,11 +35,7 @@ private: public: using SP = std::shared_ptr; RankingConstants(); - RankingConstants(RankingConstants &&) noexcept; - RankingConstants & operator =(RankingConstants &&) = delete; - RankingConstants(const RankingConstants &) = delete; - RankingConstants & operator =(const RankingConstants &) = delete; - explicit RankingConstants(const Vector &constants); + RankingConstants(const Vector &constants); ~RankingConstants(); bool operator==(const RankingConstants &rhs) const; const Constant *getConstant(const vespalib::string &name) const; diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp index 98b1e3ea653..3de6fb0cfbf 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.cpp @@ -24,7 +24,8 @@ vespalib::string extract_data(vespalib::Input &input) { } // unnamed RankingExpressions::RankingExpressions() = default; -RankingExpressions::RankingExpressions(RankingExpressions &&rhs) noexcept = default; +RankingExpressions::RankingExpressions(RankingExpressions &&rhs) = default; +RankingExpressions::RankingExpressions(const RankingExpressions &rhs) = default; RankingExpressions::~RankingExpressions() = default; RankingExpressions & diff --git a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h index c228f2b92ca..a8a7a358008 100644 --- a/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h +++ b/searchcore/src/vespa/searchcore/proton/matching/ranking_expressions.h @@ -19,13 +19,11 @@ private: std::map _expressions; public: - using SP = std::shared_ptr; RankingExpressions(); - RankingExpressions(RankingExpressions &&rhs) noexcept; - RankingExpressions & operator=(RankingExpressions &&rhs) = delete; - RankingExpressions(const RankingExpressions &rhs) = delete; - RankingExpressions & operator=(const RankingExpressions &rhs) = delete; + RankingExpressions(RankingExpressions &&rhs); + RankingExpressions(const RankingExpressions &rhs); ~RankingExpressions(); + using SP = std::shared_ptr; bool operator==(const RankingExpressions &rhs) const { return _expressions == rhs._expressions; } diff --git a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h index a58a869ca2c..79c6e929a51 100644 --- a/searchcore/src/vespa/searchcore/proton/server/buckethandler.h +++ b/searchcore/src/vespa/searchcore/proton/server/buckethandler.h @@ -45,7 +45,7 @@ public: * * @param executor The executor in which to run all tasks. */ - explicit BucketHandler(vespalib::Executor &executor); + BucketHandler(vespalib::Executor &executor); ~BucketHandler() override; void setReadyBucketHandler(documentmetastore::IBucketHandler &ready); diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp index 8be55433fdd..415000c3f9b 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.cpp @@ -54,7 +54,7 @@ blockedDueToClusterState(const std::shared_ptr &calc) } -BucketMoveJob::BucketMoveJob(std::shared_ptr calc, +BucketMoveJob::BucketMoveJob(const std::shared_ptr &calc, RetainGuard dbRetainer, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &modifiedHandler, @@ -75,7 +75,7 @@ BucketMoveJob::BucketMoveJob(std::shared_ptr calc, IBucketStateChangedHandler(), IDiskMemUsageListener(), std::enable_shared_from_this(), - _calc(std::move(calc)), + _calc(calc), _dbRetainer(std::move(dbRetainer)), _moveHandler(moveHandler), _modifiedHandler(modifiedHandler), @@ -115,7 +115,7 @@ BucketMoveJob::~BucketMoveJob() } std::shared_ptr -BucketMoveJob::create(std::shared_ptr calc, +BucketMoveJob::create(const std::shared_ptr &calc, RetainGuard dbRetainer, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &modifiedHandler, @@ -132,7 +132,7 @@ BucketMoveJob::create(std::shared_ptr calc, document::BucketSpace bucketSpace) { return std::shared_ptr( - new BucketMoveJob(std::move(calc), std::move(dbRetainer), moveHandler, modifiedHandler, master, bucketExecutor, ready, notReady, + new BucketMoveJob(calc, std::move(dbRetainer), moveHandler, modifiedHandler, master, bucketExecutor, ready, notReady, bucketCreateNotifier, clusterStateChangedNotifier, bucketStateChangedNotifier, diskMemUsageNotifier, blockableConfig, docTypeName, bucketSpace), [&master](auto job) { @@ -302,7 +302,7 @@ BucketMoveJob::considerBucket(const bucketdb::Guard & guard, BucketId bucket) { void BucketMoveJob::reconsiderBucket(const bucketdb::Guard & guard, BucketId bucket) { assert( ! _bucketsInFlight.contains(bucket)); - auto [mustMove, wantReady] = needMove(bucket, BucketStateWrapper(guard->get(bucket))); + auto [mustMove, wantReady] = needMove(bucket, guard->get(bucket)); if (mustMove) { _buckets2Move[bucket] = wantReady; } else { @@ -324,7 +324,7 @@ BucketMoveJob::computeBuckets2Move(const bucketdb::Guard & guard) BucketMoveJob::BucketMoveSet toMove; BucketId::List buckets = guard->getBuckets(); for (BucketId bucketId : buckets) { - auto [mustMove, wantReady] = needMove(bucketId, BucketStateWrapper(guard->get(bucketId))); + auto [mustMove, wantReady] = needMove(bucketId, guard->get(bucketId)); if (mustMove) { toMove[bucketId] = wantReady; } diff --git a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h index 12c205e8011..9885b581a24 100644 --- a/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h +++ b/searchcore/src/vespa/searchcore/proton/server/bucketmovejob.h @@ -83,13 +83,13 @@ private: const bucketdb::BucketState & _state; public: - explicit BucketStateWrapper(const bucketdb::BucketState & state) noexcept : _state(state) {} + BucketStateWrapper(const bucketdb::BucketState & state) noexcept : _state(state) {} bool isActive() const noexcept { return _state.isActive(); } bool hasReadyBucketDocs() const noexcept { return _state.getReadyCount() != 0; } bool hasNotReadyBucketDocs() const noexcept { return _state.getNotReadyCount() != 0; } }; - BucketMoveJob(std::shared_ptr calc, + BucketMoveJob(const std::shared_ptr &calc, vespalib::RetainGuard dbRetainer, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &modifiedHandler, @@ -124,7 +124,7 @@ private: class StartMove; public: static std::shared_ptr - create(std::shared_ptr calc, + create(const std::shared_ptr &calc, vespalib::RetainGuard dbRetainer, IDocumentMoveHandler &moveHandler, IBucketModifiedHandler &modifiedHandler, diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp index 51c02c818b2..783282b71c1 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdb.cpp @@ -74,7 +74,7 @@ constexpr uint32_t indexing_thread_stack_size = 128_Ki; index::IndexConfig makeIndexConfig(const ProtonConfig::Index & cfg) { - return {WarmupConfig(vespalib::from_s(cfg.warmup.time), cfg.warmup.unpack), size_t(cfg.maxflushed), size_t(cfg.cache.size)}; + return index::IndexConfig(WarmupConfig(vespalib::from_s(cfg.warmup.time), cfg.warmup.unpack), cfg.maxflushed, cfg.cache.size); } ReplayThrottlingPolicy @@ -92,7 +92,7 @@ make_replay_throttling_policy(const ProtonConfig::ReplayThrottlingPolicy& cfg) { class MetricsUpdateHook : public metrics::UpdateHook { DocumentDB &_db; public: - explicit MetricsUpdateHook(DocumentDB &s) + MetricsUpdateHook(DocumentDB &s) : metrics::UpdateHook("documentdb-hook"), _db(s) {} @@ -106,7 +106,7 @@ private: const DocumentDB& _doc_db; public: - explicit DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept + DocumentDBResourceUsageProvider(const DocumentDB& doc_db) noexcept : _doc_db(doc_db) {} TransientResourceUsage get_transient_resource_usage() const override { diff --git a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp index 8de142034d1..5c8abc36440 100644 --- a/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/documentdbconfigmanager.cpp @@ -210,7 +210,7 @@ deriveConfig(const ProtonConfig::Summary & summary, const HwInfo & hwInfo) { .setMaxBucketSpread(log.maxbucketspread).setMinFileSizeFactor(log.minfilesizefactor) .compactCompression(deriveCompression(log.compact.compression)) .setFileConfig(fileConfig).disableCrcOnRead(chunk.skipcrconread); - return {config, logConfig}; + return LogDocumentStore::Config(config, logConfig); } search::LogDocumentStore::Config buildStoreConfig(const ProtonConfig & proton, const HwInfo & hwInfo) { @@ -377,7 +377,7 @@ DocumentDBConfigManager::update(FNET_Transport & transport, const ConfigSnapshot OnnxModels::configure(rc, models.back()); } } - newOnnxModels = std::make_shared(std::move(models)); + newOnnxModels = std::make_shared(models); } if (snapshot.isChanged(_configId, currentGeneration)) { newIndexschemaConfig = snapshot.getConfig(_configId); diff --git a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp index cd001997c1c..49b301da26e 100644 --- a/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/maintenance_jobs_injector.cpp @@ -38,8 +38,8 @@ injectLidSpaceCompactionJobs(MaintenanceController &controller, auto job = lidspace::CompactionJob::create(config.getLidSpaceCompactionConfig(), controller.retainDB(), std::move(lidHandler), opStorer, controller.masterThread(), bucketExecutor, diskMemUsageNotifier,config.getBlockableJobConfig(), - clusterStateChangedNotifier, calc && calc->nodeRetired(), bucketSpace); - controller.registerJobInMasterThread(trackJob(std::move(tracker), std::move(job))); + clusterStateChangedNotifier, (calc ? calc->nodeRetired() : false), bucketSpace); + controller.registerJobInMasterThread(trackJob(tracker, std::move(job))); } } @@ -54,11 +54,11 @@ injectBucketMoveJob(MaintenanceController &controller, IBucketModifiedHandler &bucketModifiedHandler, IClusterStateChangedNotifier &clusterStateChangedNotifier, IBucketStateChangedNotifier &bucketStateChangedNotifier, - std::shared_ptr calc, + const std::shared_ptr &calc, DocumentDBJobTrackers &jobTrackers, IDiskMemUsageNotifier &diskMemUsageNotifier) { - auto bmj = BucketMoveJob::create(std::move(calc), controller.retainDB(), moveHandler, bucketModifiedHandler, controller.masterThread(), + auto bmj = BucketMoveJob::create(calc, controller.retainDB(), moveHandler, bucketModifiedHandler, controller.masterThread(), bucketExecutor, controller.getReadySubDB(), controller.getNotReadySubDB(), bucketCreateNotifier, clusterStateChangedNotifier, bucketStateChangedNotifier, diskMemUsageNotifier, config.getBlockableJobConfig(), docTypeName, bucketSpace); @@ -118,8 +118,7 @@ MaintenanceJobsInjector::injectJobs(MaintenanceController &controller, calc, jobTrackers, diskMemUsageNotifier); controller.registerJobInMasterThread( - std::make_unique(std::move(readyAttributeManager), - std::move(notReadyAttributeManager), + std::make_unique(readyAttributeManager, notReadyAttributeManager, attributeUsageFilter, docTypeName, config.getAttributeUsageSampleInterval())); } diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp index 6a455fb71c0..809f2e0a744 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.cpp @@ -18,10 +18,10 @@ using namespace vespalib::make_string_short; Matchers::Matchers(const vespalib::Clock &clock, matching::QueryLimiter &queryLimiter, - const matching::IRankingAssetsRepo &rankingAssetsRepo) + const matching::IConstantValueRepo &constantValueRepo) : _rpmap(), _fallback(std::make_shared(search::index::Schema(), search::fef::Properties(), clock, queryLimiter, - rankingAssetsRepo, -1)), + constantValueRepo, RankingExpressions(), OnnxModels(), -1)), _default() { } @@ -69,4 +69,14 @@ Matchers::lookup(const vespalib::string &name) const return found->second; } +vespalib::string +Matchers::listMatchers() const { + vespalib::string matchers; + for (const auto & entry : _rpmap) { + matchers += entry.first; + matchers += ' '; + } + return matchers; +} + } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/matchers.h b/searchcore/src/vespa/searchcore/proton/server/matchers.h index 72057a49935..fe467c1f565 100644 --- a/searchcore/src/vespa/searchcore/proton/server/matchers.h +++ b/searchcore/src/vespa/searchcore/proton/server/matchers.h @@ -12,7 +12,7 @@ namespace proton { namespace matching { class Matcher; class QueryLimiter; - struct IRankingAssetsRepo; + struct IConstantValueRepo; } class Matchers { @@ -22,10 +22,11 @@ private: std::shared_ptr _fallback; std::shared_ptr _default; public: - using SP = std::shared_ptr; + typedef std::shared_ptr SP; + typedef std::unique_ptr UP; Matchers(const vespalib::Clock &clock, matching::QueryLimiter &queryLimiter, - const matching::IRankingAssetsRepo &rankingAssetsRepo); + const matching::IConstantValueRepo &constantValueRepo); Matchers(const Matchers &) = delete; Matchers & operator =(const Matchers &) = delete; ~Matchers(); @@ -33,6 +34,8 @@ public: matching::MatchingStats getStats() const; matching::MatchingStats getStats(const vespalib::string &name) const; std::shared_ptr lookup(const vespalib::string &name) const; + vespalib::string listMatchers() const; + uint32_t numMatchers() const { return _rpmap.size(); } }; } // namespace proton diff --git a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp index 299eb4ac36b..e21e0366c4c 100644 --- a/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/sample_attribute_usage_job.cpp @@ -16,8 +16,8 @@ SampleAttributeUsageJob(IAttributeManagerSP readyAttributeManager, const vespalib::string &docTypeName, vespalib::duration interval) : IMaintenanceJob("sample_attribute_usage." + docTypeName, vespalib::duration::zero(), interval), - _readyAttributeManager(std::move(readyAttributeManager)), - _notReadyAttributeManager(std::move(notReadyAttributeManager)), + _readyAttributeManager(readyAttributeManager), + _notReadyAttributeManager(notReadyAttributeManager), _attributeUsageFilter(attributeUsageFilter) { } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp index f2ee50c335a..722771e4b86 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.cpp @@ -89,7 +89,7 @@ SearchableDocSubDBConfigurer(const ISummaryManager::SP &summaryMgr, SearchViewHolder &searchView, FeedViewHolder &feedView, matching::QueryLimiter &queryLimiter, - matching::RankingAssetsRepo &rankingAssetsRepo, + matching::ConstantValueRepo &constantValueRepo, const vespalib::Clock &clock, const vespalib::string &subDbName, uint32_t distributionKey) : @@ -97,7 +97,7 @@ SearchableDocSubDBConfigurer(const ISummaryManager::SP &summaryMgr, _searchView(searchView), _feedView(feedView), _queryLimiter(queryLimiter), - _rankingAssetsRepo(rankingAssetsRepo), + _constantValueRepo(constantValueRepo), _clock(clock), _subDbName(subDbName), _distributionKey(distributionKey) @@ -105,11 +105,13 @@ SearchableDocSubDBConfigurer(const ISummaryManager::SP &summaryMgr, SearchableDocSubDBConfigurer::~SearchableDocSubDBConfigurer() = default; -std::shared_ptr +Matchers::UP SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, - const RankProfilesConfig &cfg) + const RankProfilesConfig &cfg, + const RankingExpressions &rankingExpressions, + const OnnxModels &onnxModels) { - auto newMatchers = std::make_shared(_clock, _queryLimiter, _rankingAssetsRepo); + auto newMatchers = std::make_unique(_clock, _queryLimiter, _constantValueRepo); for (const auto &profile : cfg.rankprofile) { vespalib::string name = profile.name; search::fef::Properties properties; @@ -117,9 +119,9 @@ SearchableDocSubDBConfigurer::createMatchers(const Schema::SP &schema, properties.add(property.name, property.value); } // schema instance only used during call. - auto profptr = std::make_shared(*schema, std::move(properties), _clock, _queryLimiter, - _rankingAssetsRepo, _distributionKey); - newMatchers->add(name, std::move(profptr)); + auto profptr = std::make_shared(*schema, properties, _clock, _queryLimiter, _constantValueRepo, + rankingExpressions, onnxModels, _distributionKey); + newMatchers->add(name, profptr); } return newMatchers; } @@ -183,10 +185,11 @@ SearchableDocSubDBConfigurer::reconfigure(const DocumentDBConfig &newConfig, SearchView::SP searchView = _searchView.get(); Matchers::SP matchers = searchView->getMatchers(); if (params.shouldMatchersChange()) { - _rankingAssetsRepo.reconfigure(newConfig.getRankingConstantsSP(), - newConfig.getRankingExpressionsSP(), - newConfig.getOnnxModelsSP()); - Matchers::SP newMatchers = createMatchers(newConfig.getSchemaSP(), newConfig.getRankProfilesConfig()); + _constantValueRepo.reconfigure(newConfig.getRankingConstants()); + Matchers::SP newMatchers = createMatchers(newConfig.getSchemaSP(), + newConfig.getRankProfilesConfig(), + newConfig.getRankingExpressions(), + newConfig.getOnnxModels()); matchers = newMatchers; shouldMatchViewChange = true; } diff --git a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h index 6e5d6869a14..c6effd07cad 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchable_doc_subdb_configurer.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include #include #include @@ -42,7 +42,7 @@ private: SearchViewHolder &_searchView; FeedViewHolder &_feedView; matching::QueryLimiter &_queryLimiter; - matching::RankingAssetsRepo &_rankingAssetsRepo; + matching::ConstantValueRepo &_constantValueRepo; const vespalib::Clock &_clock; vespalib::string _subDbName; uint32_t _distributionKey; @@ -68,14 +68,16 @@ public: SearchViewHolder &searchView, FeedViewHolder &feedView, matching::QueryLimiter &queryLimiter, - matching::RankingAssetsRepo &rankingAssetsRepo, + matching::ConstantValueRepo &constantValueRepo, const vespalib::Clock &clock, const vespalib::string &subDbName, uint32_t distributionKey); ~SearchableDocSubDBConfigurer(); - Matchers::SP createMatchers(const search::index::Schema::SP &schema, - const vespa::config::search::RankProfilesConfig &cfg); + Matchers::UP createMatchers(const search::index::Schema::SP &schema, + const vespa::config::search::RankProfilesConfig &cfg, + const proton::matching::RankingExpressions &rankingExpressions, + const proton::matching::OnnxModels &onnxModels); void reconfigureIndexSearchable(); diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp index b623b461f6e..e366b2cbb2d 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.cpp @@ -37,8 +37,8 @@ SearchableDocSubDB::SearchableDocSubDB(const Config &cfg, const Context &ctx) _rFeedView(), _tensorLoader(FastValueBuilderFactory::get()), _constantValueCache(_tensorLoader), - _rankingAssetsRepo(_constantValueCache), - _configurer(_iSummaryMgr, _rSearchView, _rFeedView, ctx._queryLimiter, _rankingAssetsRepo, ctx._clock, + _constantValueRepo(_constantValueCache), + _configurer(_iSummaryMgr, _rSearchView, _rFeedView, ctx._queryLimiter, _constantValueRepo, ctx._clock, getSubDbName(), ctx._fastUpdCtx._storeOnlyCtx._owner.getDistributionKey()), _warmupExecutor(ctx._warmupExecutor), _realGidToLidChangeHandler(std::make_shared()), @@ -194,10 +194,9 @@ SearchableDocSubDB::initViews(const DocumentDBConfig &configSnapshot, const Sess AttributeManager::SP attrMgr = getAndResetInitAttributeManager(); const Schema::SP &schema = configSnapshot.getSchemaSP(); const IIndexManager::SP &indexMgr = getIndexManager(); - _rankingAssetsRepo.reconfigure(configSnapshot.getRankingConstantsSP(), - configSnapshot.getRankingExpressionsSP(), - configSnapshot.getOnnxModelsSP()); - Matchers::SP matchers = _configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig()); + _constantValueRepo.reconfigure(configSnapshot.getRankingConstants()); + Matchers::SP matchers = _configurer.createMatchers(schema, configSnapshot.getRankProfilesConfig(), + configSnapshot.getRankingExpressions(), configSnapshot.getOnnxModels()); auto matchView = std::make_shared(std::move(matchers), indexMgr->getSearchable(), attrMgr, sessionManager, _metaStoreCtx, _docIdLimit); _rSearchView.set(SearchView::create( diff --git a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h index 315587dd5ed..08b34a3fbe6 100644 --- a/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h +++ b/searchcore/src/vespa/searchcore/proton/server/searchabledocsubdb.h @@ -16,7 +16,7 @@ #include #include #include -#include +#include #include #include #include @@ -69,7 +69,7 @@ private: vespalib::VarHolder _rFeedView; vespalib::eval::ConstantTensorLoader _tensorLoader; vespalib::eval::ConstantValueCache _constantValueCache; - matching::RankingAssetsRepo _rankingAssetsRepo; + matching::ConstantValueRepo _constantValueRepo; SearchableDocSubDBConfigurer _configurer; vespalib::Executor &_warmupExecutor; std::shared_ptr _realGidToLidChangeHandler; -- cgit v1.2.3