diff options
26 files changed, 75 insertions, 37 deletions
diff --git a/searchlib/src/tests/query/streaming/equiv_query_node_test.cpp b/searchlib/src/tests/query/streaming/equiv_query_node_test.cpp index 72378385c78..3879b6f0d1d 100644 --- a/searchlib/src/tests/query/streaming/equiv_query_node_test.cpp +++ b/searchlib/src/tests/query/streaming/equiv_query_node_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/query/streaming/equiv_query_node.h> #include <vespa/searchlib/fef/matchdata.h> #include <vespa/searchlib/fef/simpletermdata.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/query/streaming/phrase_query_node.h> #include <vespa/searchlib/query/streaming/query.h> #include <vespa/searchlib/query/streaming/queryterm.h> @@ -15,6 +16,7 @@ using search::fef::MatchData; using search::fef::SimpleTermData; using search::fef::TermFieldHandle; using search::fef::TermFieldMatchDataPosition; +using search::fef::test::IndexEnvironment; using search::query::QueryBuilder; using search::query::Node; using search::query::SimpleQueryNodeTypes; @@ -167,7 +169,8 @@ TEST_F(EquivQueryNodeTest, test_equiv_evaluate_and_unpack) auto tfmd1 = md->resolveTermField(handle1); tfmd0->setNeedInterleavedFeatures(true); tfmd1->setNeedInterleavedFeatures(true); - eqn.unpack_match_data(2, td, *md); + IndexEnvironment ie; + eqn.unpack_match_data(2, td, *md, ie); EXPECT_EQ(2, tfmd0->getDocId()); EXPECT_EQ(3, tfmd0->getNumOccs()); EXPECT_EQ(3, tfmd0->end() - tfmd0->begin()); diff --git a/searchlib/src/tests/query/streaming_query_test.cpp b/searchlib/src/tests/query/streaming_query_test.cpp index 2129cb6805a..1aa359c9910 100644 --- a/searchlib/src/tests/query/streaming_query_test.cpp +++ b/searchlib/src/tests/query/streaming_query_test.cpp @@ -2,6 +2,7 @@ #include <vespa/searchlib/fef/simpletermdata.h> #include <vespa/searchlib/fef/matchdata.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/query/streaming/dot_product_term.h> #include <vespa/searchlib/query/streaming/equiv_query_node.h> #include <vespa/searchlib/query/streaming/in_term.h> @@ -24,6 +25,7 @@ using namespace search::streaming; using TermType = QueryTerm::Type; using search::fef::SimpleTermData; using search::fef::MatchData; +using search::fef::test::IndexEnvironment; void assertHit(const Hit & h, uint32_t exp_field_id, uint32_t exp_element_id, int32_t exp_element_weight, size_t exp_position) { EXPECT_EQ(h.field_id(), exp_field_id); @@ -774,7 +776,8 @@ TEST(StreamingQueryTest, test_in_term) q.add(12, 0, 1, 0); EXPECT_TRUE(term.evaluate()); MatchData md(MatchData::params().numTermFields(2)); - term.unpack_match_data(23, td, md); + IndexEnvironment ie; + term.unpack_match_data(23, td, md, ie); auto tmd0 = md.resolveTermField(0); EXPECT_NE(23, tmd0->getDocId()); auto tmd2 = md.resolveTermField(1); @@ -804,7 +807,8 @@ TEST(StreamingQueryTest, dot_product_term) q1.add(12, 0, 9, 0); EXPECT_TRUE(term.evaluate()); MatchData md(MatchData::params().numTermFields(2)); - term.unpack_match_data(23, td, md); + IndexEnvironment ie; + term.unpack_match_data(23, td, md, ie); auto tmd0 = md.resolveTermField(0); EXPECT_NE(23, tmd0->getDocId()); auto tmd1 = md.resolveTermField(1); @@ -849,7 +853,8 @@ check_wand_term(double limit, const vespalib::string& label) q1.add(12, 0, 4, 0); EXPECT_EQ(limit < exp_wand_score_field_11, term.evaluate()); MatchData md(MatchData::params().numTermFields(2)); - term.unpack_match_data(23, td, md); + IndexEnvironment ie; + term.unpack_match_data(23, td, md, ie); auto tmd0 = md.resolveTermField(0); EXPECT_NE(23, tmd0->getDocId()); auto tmd1 = md.resolveTermField(1); @@ -903,7 +908,8 @@ TEST(StreamingQueryTest, weighted_set_term) q1.add(12, 0, 10, 0); EXPECT_TRUE(term.evaluate()); MatchData md(MatchData::params().numTermFields(2)); - term.unpack_match_data(23, td, md); + IndexEnvironment ie; + term.unpack_match_data(23, td, md, ie); auto tmd0 = md.resolveTermField(0); EXPECT_NE(23, tmd0->getDocId()); auto tmd1 = md.resolveTermField(1); diff --git a/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.cpp b/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.cpp index 09840d9a126..bfc66d14e34 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.cpp @@ -51,7 +51,7 @@ DotProductTerm::unpack_scores(Scores& scores, std::optional<double> score_thresh } void -DotProductTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data) +DotProductTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data, const fef::IIndexEnvironment&) { Scores scores; build_scores(scores); diff --git a/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.h b/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.h index 3702bd4721c..28400b4b283 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.h +++ b/searchlib/src/vespa/searchlib/query/streaming/dot_product_term.h @@ -19,7 +19,7 @@ protected: public: DotProductTerm(std::unique_ptr<QueryNodeResultBase> result_base, const string& index, uint32_t num_terms); ~DotProductTerm() override; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.cpp index 939afec0463..3fcf983901d 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.cpp @@ -74,11 +74,11 @@ EquivQueryNode::evaluateHits(HitList & hl) const } void -EquivQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) +EquivQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) { std::vector<HitWithFieldLength> hit_list; merge_hits_from_children(hit_list, *this); - unpack_match_data_helper(docid, td, match_data, hit_list, *this); + unpack_match_data_helper(docid, td, match_data, hit_list, *this, is_filter(), index_env); } EquivQueryNode* diff --git a/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.h index b5cdb31274f..a0485954675 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.h +++ b/searchlib/src/vespa/searchlib/query/streaming/equiv_query_node.h @@ -16,7 +16,7 @@ public: ~EquivQueryNode() override; bool evaluate() const override; const HitList & evaluateHits(HitList & hl) const override; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; EquivQueryNode* as_equiv_query_node() noexcept override; const EquivQueryNode* as_equiv_query_node() const noexcept override; std::vector<std::unique_ptr<QueryTerm>> steal_terms(); diff --git a/searchlib/src/vespa/searchlib/query/streaming/in_term.cpp b/searchlib/src/vespa/searchlib/query/streaming/in_term.cpp index c164db69ba1..c856d0dd466 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/in_term.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/in_term.cpp @@ -21,7 +21,7 @@ InTerm::InTerm(std::unique_ptr<QueryNodeResultBase> result_base, const string & InTerm::~InTerm() = default; void -InTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data) +InTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data, const fef::IIndexEnvironment&) { vespalib::hash_set<uint32_t> matching_field_ids; HitList hl_store; diff --git a/searchlib/src/vespa/searchlib/query/streaming/in_term.h b/searchlib/src/vespa/searchlib/query/streaming/in_term.h index 7b388b3f6e6..8ceb8edf689 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/in_term.h +++ b/searchlib/src/vespa/searchlib/query/streaming/in_term.h @@ -14,7 +14,7 @@ public: InTerm(std::unique_ptr<QueryNodeResultBase> result_base, const string& index, std::unique_ptr<query::TermVector> terms, Normalizing normalize_mode); ~InTerm() override; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp index 07d16c7bddc..d7b1c6dc379 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.cpp @@ -52,8 +52,9 @@ NearestNeighborQueryNode::get_raw_score() const } void -NearestNeighborQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) +NearestNeighborQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) { + (void) index_env; auto raw_score = get_raw_score(); if (raw_score.has_value()) { if (td.numFields() == 1u) { diff --git a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h index 277956e27a4..7e7e513bc2e 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h +++ b/searchlib/src/vespa/searchlib/query/streaming/nearest_neighbor_query_node.h @@ -52,7 +52,7 @@ public: const std::optional<double>& get_distance() const { return _distance; } // This is used during unpacking, and also signals to the RawScoreCalculator that the entire document was a match. std::optional<double> get_raw_score() const; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.cpp b/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.cpp index b090ca13225..eef9e2e5567 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.cpp @@ -67,11 +67,11 @@ PhraseQueryNode::evaluateHits(HitList & hl) const } void -PhraseQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) +PhraseQueryNode::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) { HitList list; const HitList & hit_list = evaluateHits(list); - unpack_match_data_helper(docid, td, match_data, hit_list, *get_terms().front()); + unpack_match_data_helper(docid, td, match_data, hit_list, *get_terms().front(), is_filter(), index_env); } } diff --git a/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.h b/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.h index 0d443ac4527..615d32324be 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.h +++ b/searchlib/src/vespa/searchlib/query/streaming/phrase_query_node.h @@ -17,7 +17,7 @@ public: ~PhraseQueryNode() override; bool evaluate() const override; const HitList & evaluateHits(HitList & hl) const override; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; size_t width() const override; }; diff --git a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp index 55301132a18..611e8d67d76 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/querynode.cpp @@ -141,6 +141,7 @@ QueryNode::Build(const QueryNode * parent, const QueryNodeResultFactory & factor qt->setWeight(queryRep.GetWeight()); qt->setUniqueId(queryRep.getUniqueId()); qt->setRanked( ! queryRep.hasNoRankFlag()); + qt->set_filter(queryRep.hasNoPositionDataFlag()); if (allowRewrite && possibleFloat(*qt, ssTerm) && factory.allow_float_terms_rewrite(ssIndex)) { /* * Tokenize number term and make add alternative diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp index a9516d8aee4..728b9a2ab76 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.cpp @@ -74,6 +74,7 @@ QueryTerm::QueryTerm(std::unique_ptr<QueryNodeResultBase> org, stringref termS, _result(org.release()), _encoding(0x01), _isRanked(true), + _filter(false), _weight(100), _uniqueId(0), _fieldInfo() @@ -115,11 +116,11 @@ QueryTerm::set_element_length(uint32_t hitlist_idx, uint32_t element_length) } void -QueryTerm::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) +QueryTerm::unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) { HitList list; const HitList & hit_list = evaluateHits(list); - unpack_match_data_helper(docid, td, match_data, hit_list, *this); + unpack_match_data_helper(docid, td, match_data, hit_list, *this, is_filter(), index_env); } NearestNeighborQueryNode* diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h index 1d0454f3b63..e6b063231d6 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.h +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.h @@ -11,6 +11,7 @@ namespace search::fef { +class IIndexEnvironment; class ITermData; class MatchData; @@ -85,6 +86,8 @@ public: void setWeight(query::Weight v) { _weight = v; } void setRanked(bool ranked) { _isRanked = ranked; } bool isRanked() const { return _isRanked; } + void set_filter(bool v) noexcept { _filter = v; } + bool is_filter() const noexcept { return _filter; } void setUniqueId(uint32_t u) { _uniqueId = u; } query::Weight weight() const { return _weight; } uint32_t uniqueId() const { return _uniqueId; } @@ -105,10 +108,10 @@ public: virtual FuzzyTerm* as_fuzzy_term() noexcept; virtual EquivQueryNode* as_equiv_query_node() noexcept; virtual const EquivQueryNode* as_equiv_query_node() const noexcept; - virtual void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data); + virtual void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env); protected: template <typename HitListType> - static void unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const HitListType& hit_list, const QueryTerm& fl_term); + static void unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const HitListType& hit_list, const QueryTerm& fl_term, bool term_filter, const fef::IIndexEnvironment& index_env); using QueryNodeResultBaseContainer = std::unique_ptr<QueryNodeResultBase>; HitList _hitList; private: @@ -116,6 +119,7 @@ private: QueryNodeResultBaseContainer _result; EncodingBitMap _encoding; bool _isRanked; + bool _filter; query::Weight _weight; uint32_t _uniqueId; std::vector<FieldInfo> _fieldInfo; diff --git a/searchlib/src/vespa/searchlib/query/streaming/queryterm.hpp b/searchlib/src/vespa/searchlib/query/streaming/queryterm.hpp index dd6eff1f22b..bf10d02e6bc 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/queryterm.hpp +++ b/searchlib/src/vespa/searchlib/query/streaming/queryterm.hpp @@ -3,6 +3,7 @@ #pragma once #include "queryterm.h" +#include <vespa/searchlib/fef/iindexenvironment.h> #include <vespa/searchlib/fef/itermdata.h> #include <vespa/searchlib/fef/matchdata.h> #include <algorithm> @@ -36,7 +37,7 @@ set_interleaved_features(search::fef::TermFieldMatchData& tmd, uint32_t field_le template <typename HitListType> void -QueryTerm::unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const HitListType& hit_list, const QueryTerm& fl_term) +QueryTerm::unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const HitListType& hit_list, const QueryTerm& fl_term, bool term_filter, const fef::IIndexEnvironment& index_env) { (void) fl_term; if (!hit_list.empty()) { // only unpack if we have a hit @@ -45,6 +46,7 @@ QueryTerm::unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fe uint32_t last_field_length = 0; search::fef::TermFieldMatchData *tmd = nullptr; uint32_t num_occs = 0; + bool filter = false; // optimize for hitlist giving all hits for a single field in one chunk for (const auto& hit : hit_list) { @@ -58,6 +60,8 @@ QueryTerm::unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fe tmd = nullptr; } num_occs = 0; + auto field = index_env.getField(field_id); + filter = term_filter || (field != nullptr && field->isFilter()); // setup for new field that had a hit const search::fef::ITermFieldData *tfd = td.lookupField(field_id); @@ -77,7 +81,7 @@ QueryTerm::unpack_match_data_helper(uint32_t docid, const fef::ITermData& td, fe } } ++num_occs; - if (tmd != nullptr) { + if (tmd != nullptr && !filter) { search::fef::TermFieldMatchDataPosition pos(hit.element_id(), hit.position(), hit.element_weight(), hit.element_length()); tmd->appendPosition(pos); diff --git a/searchlib/src/vespa/searchlib/query/streaming/wand_term.cpp b/searchlib/src/vespa/searchlib/query/streaming/wand_term.cpp index a561adf5b42..a089e17adb3 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/wand_term.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/wand_term.cpp @@ -34,7 +34,7 @@ WandTerm::evaluate() const } void -WandTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data) +WandTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data, const fef::IIndexEnvironment&) { Scores scores; build_scores(scores); diff --git a/searchlib/src/vespa/searchlib/query/streaming/wand_term.h b/searchlib/src/vespa/searchlib/query/streaming/wand_term.h index 1b342834216..77041dbb256 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/wand_term.h +++ b/searchlib/src/vespa/searchlib/query/streaming/wand_term.h @@ -16,7 +16,7 @@ public: ~WandTerm() override; void set_score_threshold(double value) { _score_threshold = value; } bool evaluate() const override; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; }; } diff --git a/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.cpp b/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.cpp index d2d706eef3d..ec652e89682 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.cpp +++ b/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.cpp @@ -18,7 +18,7 @@ WeightedSetTerm::WeightedSetTerm(std::unique_ptr<QueryNodeResultBase> result_bas WeightedSetTerm::~WeightedSetTerm() = default; void -WeightedSetTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data) +WeightedSetTerm::unpack_match_data(uint32_t docid, const ITermData& td, MatchData& match_data, const fef::IIndexEnvironment&) { vespalib::hash_map<uint32_t,std::vector<double>> scores; HitList hl_store; diff --git a/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.h b/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.h index 3d8a5fba843..f90cdb20e05 100644 --- a/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.h +++ b/searchlib/src/vespa/searchlib/query/streaming/weighted_set_term.h @@ -13,7 +13,7 @@ class WeightedSetTerm : public MultiTerm { public: WeightedSetTerm(std::unique_ptr<QueryNodeResultBase> result_base, const string& index, uint32_t num_terms); ~WeightedSetTerm() override; - void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data) override; + void unpack_match_data(uint32_t docid, const fef::ITermData& td, fef::MatchData& match_data, const fef::IIndexEnvironment& index_env) override; }; } diff --git a/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp b/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp index a4a4b4e696f..ee9a08f3ecd 100644 --- a/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp +++ b/streamingvisitors/src/tests/rank_processor/rank_processor_test.cpp @@ -1,6 +1,7 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include <vespa/searchvisitor/rankprocessor.h> +#include <vespa/searchlib/fef/test/indexenvironment.h> #include <vespa/searchlib/query/streaming/query.h> #include <vespa/searchlib/query/streaming/nearest_neighbor_query_node.h> #include <vespa/searchlib/query/tree/querybuilder.h> @@ -12,6 +13,7 @@ using search::fef::MatchData; using search::fef::TermFieldHandle; using search::fef::TermFieldMatchData; +using search::fef::test::IndexEnvironment; using search::query::Weight; using search::query::QueryBuilder; using search::query::SimpleQueryNodeTypes; @@ -82,13 +84,14 @@ RankProcessorTest::test_unpack_match_data_for_term_node(bool interleaved_feature tfmd->setNeedInterleavedFeatures(interleaved_features); auto invalid_id = TermFieldMatchData::invalidId(); EXPECT_EQ(invalid_id, tfmd->getDocId()); - RankProcessor::unpack_match_data(1, *md, *_query_wrapper); + IndexEnvironment ie; + RankProcessor::unpack_match_data(1, *md, *_query_wrapper, ie); EXPECT_EQ(invalid_id, tfmd->getDocId()); node->add(field_id, 0, 1, 0); node->add(field_id, 0, 1, 1); auto& field_info = node->getFieldInfo(field_id); field_info.setFieldLength(mock_field_length); - RankProcessor::unpack_match_data(2, *md, *_query_wrapper); + RankProcessor::unpack_match_data(2, *md, *_query_wrapper, ie); EXPECT_EQ(2, tfmd->getDocId()); if (interleaved_features) { EXPECT_EQ(mock_num_occs, tfmd->getNumOccs()); @@ -99,7 +102,7 @@ RankProcessorTest::test_unpack_match_data_for_term_node(bool interleaved_feature } EXPECT_EQ(2, tfmd->size()); node->reset(); - RankProcessor::unpack_match_data(3, *md, *_query_wrapper); + RankProcessor::unpack_match_data(3, *md, *_query_wrapper, ie); EXPECT_EQ(2, tfmd->getDocId()); } @@ -145,15 +148,16 @@ TEST_F(RankProcessorTest, unpack_match_data_for_nearest_neighbor_query_node) auto tfmd = md->resolveTermField(handle); auto invalid_id = TermFieldMatchData::invalidId(); EXPECT_EQ(invalid_id, tfmd->getDocId()); - RankProcessor::unpack_match_data(1, *md, *_query_wrapper); + IndexEnvironment ie; + RankProcessor::unpack_match_data(1, *md, *_query_wrapper, ie); EXPECT_EQ(invalid_id, tfmd->getDocId()); constexpr double distance = 1.5; node->set_distance(distance); - RankProcessor::unpack_match_data(2, *md, *_query_wrapper); + RankProcessor::unpack_match_data(2, *md, *_query_wrapper, ie); EXPECT_EQ(2, tfmd->getDocId()); EXPECT_EQ(distance * 2, tfmd->getRawScore()); node->reset(); - RankProcessor::unpack_match_data(3, *md, *_query_wrapper); + RankProcessor::unpack_match_data(3, *md, *_query_wrapper, ie); EXPECT_EQ(2, tfmd->getDocId()); } diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp index 25c8d982f0f..726afcc959b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.cpp @@ -2,6 +2,7 @@ #include "indexenvironment.h" #include <vespa/searchlib/fef/i_ranking_assets_repo.h> +#include <vespa/searchlib/fef/indexproperties.h> using namespace search::fef; @@ -39,6 +40,16 @@ IndexEnvironment::addField(const vespalib::string& name, } void +IndexEnvironment::fixup_fields() +{ + for (auto& field : _fields) { + if (indexproperties::IsFilterField::check(_properties, field.name())) { + field.setFilter(true); + } + } +} + +void IndexEnvironment::set_ranking_assets_repo(std::shared_ptr<const IRankingAssetsRepo> ranking_assets_repo) { _ranking_assets_repo = std::move(ranking_assets_repo); diff --git a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h index eed38b3c922..50e6898262d 100644 --- a/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h +++ b/streamingvisitors/src/vespa/searchvisitor/indexenvironment.h @@ -78,6 +78,8 @@ public: bool isAttribute, search::fef::FieldInfo::DataType data_type); + void fixup_fields(); + search::fef::Properties & getProperties() { return _properties; } void set_ranking_assets_repo(std::shared_ptr<const search::fef::IRankingAssetsRepo> ranking_assets_repo); diff --git a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp index 7b248faaa51..3efeb8ef168 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankmanager.cpp @@ -135,6 +135,7 @@ RankManager::Snapshot::initRankSetup(const BlueprintFactory & factory) _indexEnv.push_back(_protoEnv.current()); IndexEnvironment & ie = _indexEnv.back(); ie.getProperties().import(_properties[i].second); + ie.fixup_fields(); } // set up individual rank setups per rank profile diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp index 825630f057b..2a95bf8251b 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.cpp @@ -292,16 +292,16 @@ void RankProcessor::unpackMatchData(uint32_t docId) { _docId = docId; - unpack_match_data(docId, *_match_data, _query); + unpack_match_data(docId, *_match_data, _query, _queryEnv.getIndexEnvironment()); } void -RankProcessor::unpack_match_data(uint32_t docid, MatchData &matchData, QueryWrapper& query) +RankProcessor::unpack_match_data(uint32_t docid, MatchData &matchData, QueryWrapper& query, const search::fef::IIndexEnvironment& index_env) { for (auto& term : query.getTermList()) { auto & qtd = static_cast<QueryTermData &>(term->getQueryItem()); const ITermData &td = qtd.getTermData(); - term->unpack_match_data(docid, td, matchData); + term->unpack_match_data(docid, td, matchData, index_env); } } diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index 476ed013d23..f384f7d7acf 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -74,7 +74,7 @@ public: void initForRanking(size_t wantedHitCount, bool use_sort_blob); void initForDumping(size_t wantedHitCount, bool use_sort_blob); void unpackMatchData(uint32_t docId); - static void unpack_match_data(uint32_t docid, search::fef::MatchData& matchData, QueryWrapper& query); + static void unpack_match_data(uint32_t docid, search::fef::MatchData& matchData, QueryWrapper& query, const search::fef::IIndexEnvironment& index_env); void runRankProgram(uint32_t docId); vespalib::FeatureSet::SP calculateFeatureSet(); vespalib::FeatureSet::SP calculateFeatureSet(search::DocumentIdT docId); |