diff options
27 files changed, 224 insertions, 80 deletions
diff --git a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt index 5cc2977b3c3..efe5c0507ab 100644 --- a/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt +++ b/streamingvisitors/src/tests/matching_elements_filler/CMakeLists.txt @@ -3,6 +3,7 @@ vespa_add_executable(streamingvisitors_matching_elements_filler_test_app TEST SOURCES matching_elements_filler_test.cpp DEPENDS + searchlib_test streamingvisitors GTest::GTest ) diff --git a/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp b/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp index 458031b5f01..dbe9dde469d 100644 --- a/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp +++ b/streamingvisitors/src/tests/matching_elements_filler/matching_elements_filler_test.cpp @@ -21,6 +21,7 @@ #include <vespa/vespalib/gtest/gtest.h> #include <vespa/vsm/searcher/fieldsearcher.h> #include <vespa/vsm/searcher/intfieldsearcher.h> +#include <vespa/vsm/searcher/mock_field_searcher_env.h> #include <vespa/vsm/searcher/utf8strchrfieldsearcher.h> #include <iostream> @@ -273,13 +274,12 @@ MatchingElementsFields make_matching_elements_fields() { class MatchingElementsFillerTest : public ::testing::Test { const MyDocType _doc_type; MatchingElementsFields _matching_elems_fields; - vsm::SharedFieldPathMap _field_path_map; + vsm::test::MockFieldSearcherEnv _env; vsm::FieldIdTSearcherMap _field_searcher_map; vsm::DocumentTypeIndexFieldMapT _index_to_field_ids; HitCollector _hit_collector; SearchResult _search_result; Query _query; - vsm::SharedSearcherBuf _shared_searcher_buf; std::unique_ptr<MatchingElementsFiller> _matching_elements_filler; std::unique_ptr<MatchingElements> _matching_elements; std::unique_ptr<StorageDocument> _sdoc; @@ -296,19 +296,19 @@ MatchingElementsFillerTest::MatchingElementsFillerTest() : ::testing::Test(), _doc_type(), _matching_elems_fields(make_matching_elements_fields()), - _field_path_map(make_field_path_map(_doc_type)), + _env(), _field_searcher_map(make_field_searcher_map()), _index_to_field_ids(make_index_to_field_ids()), _hit_collector(10), _search_result(), _query(), - _shared_searcher_buf(std::make_shared<vsm::SearcherBuf>()), _matching_elements_filler(), _matching_elements(), _sdoc() { + _env.field_paths = make_field_path_map(_doc_type); _search_result.addHit(1, "id::test::1", 0.0, nullptr, 0); - _sdoc = std::make_unique<StorageDocument>(_doc_type.make_test_doc(), _field_path_map, _field_path_map->size()); + _sdoc = std::make_unique<StorageDocument>(_doc_type.make_test_doc(), _env.field_paths, _env.field_paths->size()); EXPECT_TRUE(_sdoc->valid()); MatchData md(MatchData::params()); _hit_collector.addHit(_sdoc.get(), 1, md, 0.0, nullptr, 0); @@ -322,7 +322,7 @@ MatchingElementsFillerTest::fill_matching_elements(Query &&query) _matching_elements_filler.reset(); _matching_elements.reset(); _query = std::move(query); - _field_searcher_map.prepare(_index_to_field_ids, _shared_searcher_buf, _query); + _env.prepare(_field_searcher_map, _index_to_field_ids, _query); _matching_elements_filler = std::make_unique<MatchingElementsFiller>(_field_searcher_map, _query, _hit_collector, _search_result); _matching_elements = _matching_elements_filler->fill_matching_elements(_matching_elems_fields); } diff --git a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp index 70c3ae1002f..43c77398be8 100644 --- a/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp +++ b/streamingvisitors/src/tests/nearest_neighbor_field_searcher/nearest_neighbor_field_searcher_test.cpp @@ -1,5 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +#include <vespa/document/base/fieldpath.h> #include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/eval/eval/simple_value.h> @@ -7,14 +8,10 @@ #include <vespa/eval/eval/value_codec.h> #include <vespa/eval/eval/value_type.h> #include <vespa/searchlib/fef/indexproperties.h> -#include <vespa/searchlib/fef/properties.h> -#include <vespa/searchlib/fef/tablemanager.h> #include <vespa/searchlib/query/streaming/nearest_neighbor_query_node.h> #include <vespa/searchlib/tensor/euclidean_distance.h> -#include <vespa/searchlib/test/mock_attribute_manager.h> -#include <vespa/searchvisitor/indexenvironment.h> -#include <vespa/searchvisitor/queryenvironment.h> #include <vespa/vespalib/gtest/gtest.h> +#include <vespa/vsm/searcher/mock_field_searcher_env.h> #include <vespa/vsm/searcher/nearest_neighbor_field_searcher.h> using namespace search::attribute::test; @@ -25,8 +22,10 @@ using namespace search::tensor; using namespace vespalib::eval; using namespace vsm; -using document::TensorFieldValue; +using document::FieldPath; +using document::FieldPathEntry; using document::TensorDataType; +using document::TensorFieldValue; struct MockQuery { std::vector<std::unique_ptr<NearestNeighborQueryNode>> nodes; @@ -53,41 +52,36 @@ struct MockQuery { class NearestNeighborSearcherTest : public testing::Test { public: - TableManager table_mgr; - streaming::IndexEnvironment index_env; - MockAttributeManager attr_mgr; - Properties query_props; - streaming::QueryEnvironment query_env; + vsm::test::MockFieldSearcherEnv env; ValueType tensor_type; TensorDataType data_type; SquaredEuclideanDistance dist_func; + vsm::FieldIdT field_id; NearestNeighborFieldSearcher searcher; MockQuery query; NearestNeighborSearcherTest() - : table_mgr(), - index_env(table_mgr), - attr_mgr(), - query_props(), - query_env("", index_env, query_props, &attr_mgr), + : env(), tensor_type(ValueType::from_spec("tensor(x[2])")), data_type(tensor_type), dist_func(CellType::DOUBLE), - searcher(7, DistanceMetric::Euclidean), + field_id(2), + searcher(field_id, DistanceMetric::Euclidean), query() { + env.field_paths->resize(field_id + 1); + (*env.field_paths)[field_id].push_back(std::make_unique<FieldPathEntry>(data_type, "my_tensor_field")); } void set_query_tensor(const vespalib::string& query_tensor_name, const vespalib::string& spec_expr) { - search::fef::indexproperties::type::QueryFeature::set(index_env.getProperties(), query_tensor_name, tensor_type.to_spec()); + search::fef::indexproperties::type::QueryFeature::set(env.index_env.getProperties(), query_tensor_name, tensor_type.to_spec()); auto tensor = SimpleValue::from_spec(TensorSpec::from_expr(spec_expr)); vespalib::nbostream stream; vespalib::eval::encode_value(*tensor, stream); - query_props.add(query_tensor_name, vespalib::stringref(stream.peek(), stream.size())); + env.query_props.add(query_tensor_name, vespalib::stringref(stream.peek(), stream.size())); } void prepare() { - auto searcher_buf = std::make_shared<SearcherBuf>(); - searcher.prepare_new(query.term_list, searcher_buf, tensor_type, query_env); + env.prepare(searcher, query.term_list); } void match(const vespalib::string& spec_expr) { TensorFieldValue fv(data_type); diff --git a/streamingvisitors/src/tests/searcher/CMakeLists.txt b/streamingvisitors/src/tests/searcher/CMakeLists.txt index 2277f5ef55f..d43c1b58d33 100644 --- a/streamingvisitors/src/tests/searcher/CMakeLists.txt +++ b/streamingvisitors/src/tests/searcher/CMakeLists.txt @@ -3,6 +3,8 @@ vespa_add_executable(vsm_searcher_test_app TEST SOURCES searcher_test.cpp DEPENDS + searchlib + searchlib_test streamingvisitors ) vespa_add_test(NAME vsm_searcher_test_app COMMAND vsm_searcher_test_app) diff --git a/streamingvisitors/src/tests/searcher/searcher_test.cpp b/streamingvisitors/src/tests/searcher/searcher_test.cpp index 607e418ac83..6c4ac69298e 100644 --- a/streamingvisitors/src/tests/searcher/searcher_test.cpp +++ b/streamingvisitors/src/tests/searcher/searcher_test.cpp @@ -2,19 +2,20 @@ #include <vespa/vespalib/testkit/testapp.h> +#include <vespa/document/fieldvalue/fieldvalues.h> +#include <vespa/searchlib/query/streaming/queryterm.h> +#include <vespa/vsm/searcher/boolfieldsearcher.h> #include <vespa/vsm/searcher/fieldsearcher.h> #include <vespa/vsm/searcher/floatfieldsearcher.h> #include <vespa/vsm/searcher/futf8strchrfieldsearcher.h> #include <vespa/vsm/searcher/intfieldsearcher.h> -#include <vespa/vsm/searcher/boolfieldsearcher.h> -#include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h> +#include <vespa/vsm/searcher/mock_field_searcher_env.h> #include <vespa/vsm/searcher/utf8exactstringfieldsearcher.h> +#include <vespa/vsm/searcher/utf8flexiblestringfieldsearcher.h> #include <vespa/vsm/searcher/utf8substringsearcher.h> #include <vespa/vsm/searcher/utf8substringsnippetmodifier.h> #include <vespa/vsm/searcher/utf8suffixstringfieldsearcher.h> #include <vespa/vsm/vsm/snippetmodifier.h> -#include <vespa/searchlib/query/streaming/queryterm.h> -#include <vespa/document/fieldvalue/fieldvalues.h> using namespace document; using search::streaming::HitList; @@ -102,7 +103,7 @@ struct SnippetModifierSetup { Query query; UTF8SubstringSnippetModifier::SP searcher; - SharedSearcherBuf buf; + test::MockFieldSearcherEnv env; SnippetModifier modifier; explicit SnippetModifierSetup(const StringList & terms); ~SnippetModifierSetup(); @@ -111,10 +112,10 @@ struct SnippetModifierSetup SnippetModifierSetup::SnippetModifierSetup(const StringList & terms) : query(terms), searcher(new UTF8SubstringSnippetModifier()), - buf(new SearcherBuf(8)), + env(), modifier(searcher) { - searcher->prepare(query.qtl, buf); + env.prepare(*searcher, query.qtl); } SnippetModifierSetup::~SnippetModifierSetup() = default; @@ -310,8 +311,8 @@ performSearch(FieldSearcher & fs, const StringList & query, const FieldValue & f Query q(query); // prepare field searcher - SharedSearcherBuf ssb = SharedSearcherBuf(new SearcherBuf()); - fs.prepare(q.qtl, ssb); + test::MockFieldSearcherEnv env; + env.prepare(fs, q.qtl); // setup document SharedFieldPathMap sfim(new FieldPathMapT()); @@ -786,39 +787,40 @@ TEST("snippet modifier manager") { indexMap["i1"].push_back(1); indexMap["i2"].push_back(0); indexMap["i2"].push_back(1); + test::MockFieldSearcherEnv env; { SnippetModifierManager man; Query query(StringList().add("i0:foo")); - man.setup(query.qtl, specMap, indexMap); + man.setup(query.qtl, specMap, indexMap, *env.field_paths, env.query_env); assertQueryTerms(man, 0, StringList().add("foo")); assertQueryTerms(man, 1, StringList()); } { SnippetModifierManager man; Query query(StringList().add("i1:foo")); - man.setup(query.qtl, specMap, indexMap); + man.setup(query.qtl, specMap, indexMap, *env.field_paths, env.query_env); assertQueryTerms(man, 0, StringList()); assertQueryTerms(man, 1, StringList()); } { SnippetModifierManager man; Query query(StringList().add("i1:*foo*")); - man.setup(query.qtl, specMap, indexMap); + man.setup(query.qtl, specMap, indexMap, *env.field_paths, env.query_env); assertQueryTerms(man, 0, StringList()); assertQueryTerms(man, 1, StringList().add("foo")); } { SnippetModifierManager man; Query query(StringList().add("i2:foo").add("i2:*bar*")); - man.setup(query.qtl, specMap, indexMap); + man.setup(query.qtl, specMap, indexMap, *env.field_paths, env.query_env); assertQueryTerms(man, 0, StringList().add("foo").add("bar")); assertQueryTerms(man, 1, StringList().add("bar")); } { // check buffer sizes SnippetModifierManager man; Query query(StringList().add("i2:foo").add("i2:*bar*")); - man.setup(query.qtl, specMap, indexMap); + man.setup(query.qtl, specMap, indexMap, *env.field_paths, env.query_env); { SnippetModifier * sm = static_cast<SnippetModifier *>(man.getModifiers().getModifier(0)); UTF8SubstringSnippetModifier * searcher = sm->getSearcher().get(); diff --git a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h index 039a9386539..c541f62646e 100644 --- a/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h +++ b/streamingvisitors/src/vespa/searchvisitor/rankprocessor.h @@ -72,6 +72,7 @@ public: double getRankScore() const { return _score; } HitCollector & getHitCollector() { return *_hitCollector; } uint32_t getDocId() const { return _docId; } + search::fef::IQueryEnvironment& get_query_env() { return _queryEnv; } }; } // namespace streaming diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp index 2caf2de1d0b..7dc0c05cfaa 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.cpp @@ -387,7 +387,6 @@ void SearchVisitor::init(const Parameters & params) StringFieldIdTMap fieldsInQuery; setupFieldSearchers(additionalFields, fieldsInQuery); - setupSnippetModifiers(); setupScratchDocument(fieldsInQuery); @@ -400,9 +399,18 @@ void SearchVisitor::init(const Parameters & params) const RankManager * rm = _env.getRankManager(searchCluster); _rankController.setRankManagerSnapshot(rm->getSnapshot()); _rankController.setupRankProcessors(_query, location, wantedSummaryCount, _attrMan, _attributeFields); - // Depends on hitCollector setup. + + // This depends on _fieldPathMap (from setupScratchDocument), + // and IQueryEnvironment (from setupRankProcessors). + setupSnippetModifiers(); + + // Depends on hitCollector setup and _snippetModifierManager setupDocsumObjects(); + // This depends on _fieldPathMap (from setupScratchDocument), + // and IQueryEnvironment (from setupRankProcessors). + prepare_field_searchers(); + } else { LOG(warning, "No query received"); } @@ -732,9 +740,14 @@ SearchVisitor::setupFieldSearchers(const std::vector<vespalib::string> & additio _fieldSearchSpecMap.buildFieldsInQuery(_query, fieldsInQuery); // Connect field names in the query to field searchers _fieldSearchSpecMap.buildSearcherMap(fieldsInQuery.map(), _fieldSearcherMap); +} +void +SearchVisitor::prepare_field_searchers() +{ // prepare the field searchers - _fieldSearcherMap.prepare(_fieldSearchSpecMap.documentTypeMap(), _searchBuffer, _query); + _fieldSearcherMap.prepare(_fieldSearchSpecMap.documentTypeMap(), _searchBuffer, _query, + *_fieldPathMap, _rankController.getRankProcessor()->get_query_env()); } void @@ -742,7 +755,8 @@ SearchVisitor::setupSnippetModifiers() { QueryTermList qtl; _query.getLeafs(qtl); - _snippetModifierManager.setup(qtl, _fieldSearchSpecMap.specMap(), _fieldSearchSpecMap.documentTypeMap().begin()->second); + _snippetModifierManager.setup(qtl, _fieldSearchSpecMap.specMap(), _fieldSearchSpecMap.documentTypeMap().begin()->second, + *_fieldPathMap, _rankController.getRankProcessor()->get_query_env()); } void diff --git a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h index 06ab2eaeaaa..162c031ac9e 100644 --- a/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h +++ b/streamingvisitors/src/vespa/searchvisitor/searchvisitor.h @@ -260,6 +260,13 @@ private: vsm::StringFieldIdTMap & fieldsInQuery); /** + * Prepare the field searchers for the given query. + * This includes connecting the query terms searching a given field to that field searcher, + * and setting up objects in the field searcher needed when matching later on. + **/ + void prepare_field_searchers(); + + /** * Setup snippet modifiers for the fields where we have substring search. * The modifiers will be used when generating docsum. **/ diff --git a/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.cpp index 8c9b556e593..47b88429f06 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.cpp @@ -25,10 +25,13 @@ BoolFieldSearcher::BoolFieldSearcher(FieldIdT fId) : BoolFieldSearcher::~BoolFieldSearcher() = default; -void BoolFieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) +void BoolFieldSearcher::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { _terms.clear(); - FieldSearcher::prepare(qtl, buf); + FieldSearcher::prepare(qtl, buf, field_paths, query_env); for (const QueryTerm * qt : qtl) { if (TRUE == qt->getTerm()) { _terms.push_back(true); diff --git a/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.h index f6afef9e507..01c907888c6 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/boolfieldsearcher.h @@ -11,7 +11,10 @@ public: std::unique_ptr<FieldSearcher> duplicate() const override; BoolFieldSearcher(FieldIdT fId); ~BoolFieldSearcher(); - void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; void onValue(const document::FieldValue & fv) override; private: std::vector<bool> _terms; diff --git a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp index aad4f9d5aa2..cbf8903caab 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.cpp @@ -105,7 +105,10 @@ bool FieldSearcher::search(const StorageDocument & doc) return true; } -void FieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf &) +void FieldSearcher::prepare(QueryTermList& qtl, + const SharedSearcherBuf&, + const vsm::FieldPathMapT&, + search::fef::IQueryEnvironment&) { FieldSearcherBase::prepare(qtl); prepareFieldId(); @@ -220,7 +223,11 @@ void FieldSearcher::init() } } -void FieldIdTSearcherMap::prepare(const DocumentTypeIndexFieldMapT & difm, const SharedSearcherBuf & searcherBuf, Query & query) +void FieldIdTSearcherMap::prepare(const DocumentTypeIndexFieldMapT& difm, + const SharedSearcherBuf& searcherBuf, + Query& query, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { QueryTermList qtl; query.getLeafs(qtl); @@ -244,7 +251,7 @@ void FieldIdTSearcherMap::prepare(const DocumentTypeIndexFieldMapT & difm, const } } /// Should perhaps do a unique on onlyInIndex - (*it)->prepare(onlyInIndex, searcherBuf); + (*it)->prepare(onlyInIndex, searcherBuf, field_paths, query_env); if (LOG_WOULD_LOG(spam)) { char tmpBuf[16]; snprintf(tmpBuf, sizeof(tmpBuf), "%d", fid); diff --git a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h index 55aabf3c0cf..879902ca514 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/fieldsearcher.h @@ -7,6 +7,8 @@ #include <vespa/vsm/common/storagedocument.h> #include <vespa/vespalib/util/array.h> +namespace search::fef { class IQueryEnvironment; } + namespace vsm { using termcount_t = size_t; @@ -54,7 +56,11 @@ public: ~FieldSearcher() override; virtual std::unique_ptr<FieldSearcher> duplicate() const = 0; bool search(const StorageDocument & doc); - virtual void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf); + virtual void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env); + const FieldIdT & field() const { return _field; } void field(const FieldIdT & v) { _field = v; prepareFieldId(); } bool prefix() const { return _matchType == PREFIX; } @@ -142,7 +148,11 @@ using FieldIdTSearcherMapT = std::vector<FieldSearcherContainer>; class FieldIdTSearcherMap : public FieldIdTSearcherMapT { public: - void prepare(const DocumentTypeIndexFieldMapT & difm, const SharedSearcherBuf & searcherBuf, search::streaming::Query & query); + void prepare(const DocumentTypeIndexFieldMapT& difm, + const SharedSearcherBuf& searcherBuf, + search::streaming::Query& query, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env); }; } diff --git a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp index 02d8bd8c12a..8585975ca3c 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.cpp @@ -29,10 +29,13 @@ template<typename T> FloatFieldSearcherT<T>::~FloatFieldSearcherT() {} template<typename T> -void FloatFieldSearcherT<T>::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) +void FloatFieldSearcherT<T>::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { _floatTerm.clear(); - FieldSearcher::prepare(qtl, buf); + FieldSearcher::prepare(qtl, buf, field_paths, query_env); for (QueryTermList::const_iterator it=qtl.begin(); it < qtl.end(); it++) { const QueryTerm * qt = *it; size_t sz(qt->termLen()); diff --git a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.h index 80f79fd6491..a25f7a74d64 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/floatfieldsearcher.h @@ -11,7 +11,10 @@ class FloatFieldSearcherT : public FieldSearcher public: FloatFieldSearcherT(FieldIdT fId=0); ~FloatFieldSearcherT(); - void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; void onValue(const document::FieldValue & fv) override; protected: class FloatInfo diff --git a/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp index db93bda7778..6b0bbbb368d 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.cpp @@ -28,9 +28,13 @@ GeoPosFieldSearcher::GeoPosFieldSearcher(FieldIdT fId) : GeoPosFieldSearcher::~GeoPosFieldSearcher() {} -void GeoPosFieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) { +void GeoPosFieldSearcher::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) +{ _geoPosTerm.clear(); - FieldSearcher::prepare(qtl, buf); + FieldSearcher::prepare(qtl, buf, field_paths, query_env); for (const QueryTerm * qt : qtl) { const vespalib::string & str = qt->getTermString(); GeoLocationParser parser; diff --git a/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.h b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.h index 307be1dacab..b3d09594c01 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/geo_pos_field_searcher.h @@ -10,7 +10,10 @@ class GeoPosFieldSearcher : public FieldSearcher { public: GeoPosFieldSearcher(FieldIdT fId=0); ~GeoPosFieldSearcher(); - void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; void onValue(const document::FieldValue & fv) override; void onStructValue(const document::StructFieldValue & fv) override; std::unique_ptr<FieldSearcher> duplicate() const override; diff --git a/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp index 8cfb8e6df14..18b286946f7 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.cpp @@ -19,10 +19,13 @@ IntFieldSearcher::IntFieldSearcher(FieldIdT fId) : IntFieldSearcher::~IntFieldSearcher() = default; -void IntFieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) +void IntFieldSearcher::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { _intTerm.clear(); - FieldSearcher::prepare(qtl, buf); + FieldSearcher::prepare(qtl, buf, field_paths, query_env); for (QueryTermList::const_iterator it=qtl.begin(); it < qtl.end(); it++) { const QueryTerm * qt = *it; size_t sz(qt->termLen()); diff --git a/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.h index 6645a5d719a..f9ae098dae5 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/intfieldsearcher.h @@ -11,7 +11,10 @@ public: std::unique_ptr<FieldSearcher> duplicate() const override; IntFieldSearcher(FieldIdT fId=0); ~IntFieldSearcher(); - void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; void onValue(const document::FieldValue & fv) override; protected: class IntInfo diff --git a/streamingvisitors/src/vespa/vsm/searcher/mock_field_searcher_env.h b/streamingvisitors/src/vespa/vsm/searcher/mock_field_searcher_env.h new file mode 100644 index 00000000000..0c872ad120f --- /dev/null +++ b/streamingvisitors/src/vespa/vsm/searcher/mock_field_searcher_env.h @@ -0,0 +1,48 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. + +#pragma once + +#include "fieldsearcher.h" +#include <vespa/searchlib/fef/properties.h> +#include <vespa/searchlib/fef/tablemanager.h> +#include <vespa/searchlib/test/mock_attribute_manager.h> +#include <vespa/searchvisitor/indexenvironment.h> +#include <vespa/searchvisitor/queryenvironment.h> +#include <vespa/vsm/common/storagedocument.h> + +namespace vsm::test { + +/** + * Mock of the objects needed to prepare a FieldSearcher. + * Only used for unit testing. + */ +struct MockFieldSearcherEnv { + SharedSearcherBuf buf; + vsm::SharedFieldPathMap field_paths; + search::fef::TableManager table_mgr; + streaming::IndexEnvironment index_env; + search::attribute::test::MockAttributeManager attr_mgr; + search::fef::Properties query_props; + streaming::QueryEnvironment query_env; + MockFieldSearcherEnv() + : buf(new SearcherBuf(8)), + field_paths(std::make_shared<FieldPathMapT>()), + table_mgr(), + index_env(table_mgr), + attr_mgr(), + query_props(), + query_env("", index_env, query_props, &attr_mgr) + {} + ~MockFieldSearcherEnv() {} + void prepare(FieldSearcher& searcher, search::streaming::QueryTermList& qtl) { + searcher.prepare(qtl, buf, *field_paths, query_env); + } + void prepare(FieldIdTSearcherMap& searcher_map, + const DocumentTypeIndexFieldMapT& difm, + search::streaming::Query& query) { + searcher_map.prepare(difm, buf, query, *field_paths, query_env); + } +}; + +} + diff --git a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp index 82243d67e5a..045ec9b04a3 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.cpp @@ -1,6 +1,8 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "nearest_neighbor_field_searcher.h" +#include <vespa/document/datatype/datatype.h> +#include <vespa/document/datatype/tensor_data_type.h> #include <vespa/document/fieldvalue/tensorfieldvalue.h> #include <vespa/searchcommon/attribute/config.h> #include <vespa/searchlib/fef/iqueryenvironment.h> @@ -59,6 +61,8 @@ NearestNeighborFieldSearcher::NearestNeighborFieldSearcher(const FieldIdT& fid, { } +NearestNeighborFieldSearcher::~NearestNeighborFieldSearcher() = default; + std::unique_ptr<FieldSearcher> NearestNeighborFieldSearcher::duplicate() const { @@ -66,10 +70,18 @@ NearestNeighborFieldSearcher::duplicate() const } void -NearestNeighborFieldSearcher::prepare_new(search::streaming::QueryTermList& qtl, const SharedSearcherBuf&, - const vespalib::eval::ValueType& tensor_type, search::fef::IQueryEnvironment& query_env) +NearestNeighborFieldSearcher::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { - _attr = make_attribute(tensor_type); + FieldSearcher::prepare(qtl, buf, field_paths, query_env); + const auto* tensor_type = field_paths[field()].back().getDataType().cast_tensor(); + if (tensor_type == nullptr) { + vespalib::Issue::report("Data type for field %u is '%s', but expected it to be a tensor type", + field(), field_paths[field()].back().getDataType().toString().c_str()); + } + _attr = make_attribute(tensor_type->getTensorType()); _calcs.clear(); for (auto term : qtl) { auto* nn_term = term->as_nearest_neighbor_query_node(); diff --git a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h index ec78360b216..83f2c444e5a 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/nearest_neighbor_field_searcher.h @@ -40,11 +40,13 @@ private: public: NearestNeighborFieldSearcher(const FieldIdT& fid, search::attribute::DistanceMetric metric); + ~NearestNeighborFieldSearcher(); std::unique_ptr<FieldSearcher> duplicate() const override; - // TODO: change FieldSearcher::prepare() to provide the necessary objects. - void prepare_new(search::streaming::QueryTermList& qtl, const SharedSearcherBuf& buf, - const vespalib::eval::ValueType& tensor_type, search::fef::IQueryEnvironment& query_env); + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; void onValue(const document::FieldValue& fv) override; static search::attribute::DistanceMetric distance_metric_from_string(const vespalib::string& value); diff --git a/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp index 1c4ff78ff4a..f15290526d9 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.cpp @@ -7,9 +7,12 @@ using search::streaming::QueryTermList; namespace vsm { -void StrChrFieldSearcher::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) +void StrChrFieldSearcher::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { - FieldSearcher::prepare(qtl, buf); + FieldSearcher::prepare(qtl, buf, field_paths, query_env); } void StrChrFieldSearcher::onValue(const document::FieldValue & fv) diff --git a/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.h b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.h index 0155c79cddf..c155aaefea9 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.h +++ b/streamingvisitors/src/vespa/vsm/searcher/strchrfieldsearcher.h @@ -11,7 +11,10 @@ public: StrChrFieldSearcher() : FieldSearcher(0) { } StrChrFieldSearcher(FieldIdT fId) : FieldSearcher(fId) { } void onValue(const document::FieldValue & fv) override; - void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; private: size_t shortestTerm() const; bool matchDoc(const FieldRef & field); diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp index 148cdf2c0c3..f991722d623 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.cpp @@ -236,9 +236,12 @@ UTF8StringFieldSearcherBase::UTF8StringFieldSearcherBase(FieldIdT fId) : UTF8StringFieldSearcherBase::~UTF8StringFieldSearcherBase() {} void -UTF8StringFieldSearcherBase::prepare(QueryTermList & qtl, const SharedSearcherBuf & buf) +UTF8StringFieldSearcherBase::prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { - StrChrFieldSearcher::prepare(qtl, buf); + StrChrFieldSearcher::prepare(qtl, buf, field_paths, query_env); _buf = buf; } diff --git a/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h index f540a7ac457..a017b501660 100644 --- a/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h +++ b/streamingvisitors/src/vespa/vsm/searcher/utf8stringfieldsearcherbase.h @@ -107,7 +107,10 @@ public: UTF8StringFieldSearcherBase(); UTF8StringFieldSearcherBase(FieldIdT fId); ~UTF8StringFieldSearcherBase(); - void prepare(search::streaming::QueryTermList & qtl, const SharedSearcherBuf & buf) override; + void prepare(search::streaming::QueryTermList& qtl, + const SharedSearcherBuf& buf, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) override; /** * Matches the given query term against the given word using suffix match strategy. * diff --git a/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.cpp b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.cpp index 1993f1cf657..2d2d3f24bc6 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.cpp +++ b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.cpp @@ -99,9 +99,11 @@ SnippetModifierManager::SnippetModifierManager() : SnippetModifierManager::~SnippetModifierManager() {} void -SnippetModifierManager::setup(const QueryTermList & queryTerms, - const FieldSearchSpecMapT & specMap, - const IndexFieldMapT & indexMap) +SnippetModifierManager::setup(const QueryTermList& queryTerms, + const FieldSearchSpecMapT& specMap, + const IndexFieldMapT& indexMap, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env) { FieldQueryTermMap fqtm; @@ -130,7 +132,7 @@ SnippetModifierManager::setup(const QueryTermList & queryTerms, for (auto & entry : _modifiers.map()) { FieldIdT fId = entry.first; SnippetModifier & smod = static_cast<SnippetModifier &>(*entry.second); - smod.getSearcher()->prepare(fqtm[fId], _searchBuf); + smod.getSearcher()->prepare(fqtm[fId], _searchBuf, field_paths, query_env); } } diff --git a/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.h b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.h index 1f8cb952948..c6ca115bda2 100644 --- a/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.h +++ b/streamingvisitors/src/vespa/vsm/vsm/snippetmodifier.h @@ -99,9 +99,14 @@ public: * @param queryTerms the query terms to take into consideration. * @param specMap mapping from field id to search spec objects. * @param fieldMap mapping from index (used in the query) to a list of field ids. + * @param field_paths mapping from field id to document::FieldPath. + * @param query_env query environment containg e.g. query tensors. **/ - void setup(const search::streaming::QueryTermList & queryTerms, - const FieldSearchSpecMapT & specMap, const IndexFieldMapT & fieldMap); + void setup(const search::streaming::QueryTermList& queryTerms, + const FieldSearchSpecMapT& specMap, + const IndexFieldMapT& fieldMap, + const vsm::FieldPathMapT& field_paths, + search::fef::IQueryEnvironment& query_env); const FieldModifierMap & getModifiers() const { return _modifiers; } }; |