diff options
author | Henning Baldersheim <balder@yahoo-inc.com> | 2024-02-05 23:00:27 +0000 |
---|---|---|
committer | Henning Baldersheim <balder@yahoo-inc.com> | 2024-02-05 23:00:27 +0000 |
commit | d59d1cdb2b4872c2309cfad2e96012fdbdfc6ff9 (patch) | |
tree | 64bf053dd750d9ae2c1ec5f9ce7500d0945f22ee /searchsummary | |
parent | c48eb091494ccb39d2edd0a1b50073f3c5dc4c2b (diff) |
Wire QueryNormalization in to JuniperQueryAdapter and use it there.
Diffstat (limited to 'searchsummary')
6 files changed, 34 insertions, 20 deletions
diff --git a/searchsummary/src/vespa/juniper/queryvisitor.cpp b/searchsummary/src/vespa/juniper/queryvisitor.cpp index bd18b548ab7..6192cb985b9 100644 --- a/searchsummary/src/vespa/juniper/queryvisitor.cpp +++ b/searchsummary/src/vespa/juniper/queryvisitor.cpp @@ -282,8 +282,7 @@ QueryVisitor::visitKeyword(const QueryItem* item, vespalib::stringref keyword, b } -namespace juniper -{ +namespace juniper { const char* creator_text(ItemCreator creator) { diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp index 9aa327f8e7b..c0a0c79c0ca 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.cpp @@ -43,6 +43,7 @@ GetDocsumsState::GetDocsumsState(GetDocsumsStateCallback &callback) _attrCtx(), _attributes(), _stash(), + _normalization(nullptr), _fieldWriterStates(), _parsedLocations(), _summaryFeatures(nullptr), diff --git a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h index f2b80fa2886..c01309961ed 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/docsumstate.h @@ -4,6 +4,7 @@ #include "getdocsumargs.h" #include <vespa/searchlib/common/geo_location_spec.h> +#include <vespa/searchlib/query/query_normalization.h> #include <vespa/vespalib/stllike/hash_map.h> #include <vespa/vespalib/util/featureset.h> #include <vespa/vespalib/util/stash.h> @@ -67,7 +68,8 @@ public: std::unique_ptr<search::attribute::IAttributeContext> _attrCtx; std::vector<const search::attribute::IAttributeVector *> _attributes; private: - vespalib::Stash _stash; + vespalib::Stash _stash; + const QueryNormalization *_normalization; public: // DocsumFieldWriterState instances are owned by _stash std::vector<DocsumFieldWriterState*> _fieldWriterStates; @@ -94,6 +96,8 @@ public: const MatchingElements &get_matching_elements(const MatchingElementsFields &matching_elems_fields); vespalib::Stash& get_stash() noexcept { return _stash; } + const QueryNormalization * query_normalization() const { return _normalization; } + void query_normalization(const QueryNormalization * normalization) { _normalization = normalization; } }; } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp index 9be6a2d8d7c..66b6211f61e 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/dynamicteaserdfw.cpp @@ -37,9 +37,8 @@ DynamicTeaserDFW::insert_juniper_field(uint32_t docid, vespalib::stringref input { auto& query = state._dynteaser.get_query(_input_field_name); if (!query) { - JuniperQueryAdapter iq(_query_term_filter.get(), - state._args.getStackDump(), - state._args.highlightTerms()); + JuniperQueryAdapter iq(state.query_normalization(), _query_term_filter.get(), + state._args.getStackDump(), state._args.highlightTerms()); query = _juniper->CreateQueryHandle(iq, nullptr); } diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp b/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp index 1131b94d09a..03979675c99 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.cpp @@ -7,12 +7,14 @@ #include <vespa/searchlib/fef/properties.h> #include <vespa/searchlib/parsequery/stackdumpiterator.h> #include <vespa/searchlib/queryeval/split_float.h> +#include <vespa/searchlib/query/query_normalization.h> namespace search::docsummary { -JuniperQueryAdapter::JuniperQueryAdapter(const IQueryTermFilter *query_term_filter, vespalib::stringref buf, - const search::fef::Properties & highlightTerms) - : _query_term_filter(query_term_filter), +JuniperQueryAdapter::JuniperQueryAdapter(const QueryNormalization * normalization, const IQueryTermFilter *query_term_filter, + vespalib::stringref buf, const search::fef::Properties & highlightTerms) + : _query_normalization(normalization), + _query_term_filter(query_term_filter), _buf(buf), _highlightTerms(highlightTerms) { @@ -47,6 +49,7 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const } while (rc && iterator.next()) { bool isSpecialToken = iterator.hasSpecialTokenFlag(); + bool prefix_like = false; switch (iterator.getType()) { case search::ParseItem::ITEM_OR: case search::ParseItem::ITEM_WEAK_AND: @@ -67,12 +70,23 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const if (!v->VisitRANK(&item, iterator.getArity())) rc = skipItem(&iterator); break; + case search::ParseItem::ITEM_PREFIXTERM: + case search::ParseItem::ITEM_SUBSTRINGTERM: + prefix_like = true; + [[fallthrough]]; case search::ParseItem::ITEM_TERM: case search::ParseItem::ITEM_EXACTSTRINGTERM: case search::ParseItem::ITEM_PURE_WEIGHTED_STRING: { - vespalib::stringref term = iterator.getTerm(); - v->visitKeyword(&item, term, false, isSpecialToken); + vespalib::string term = iterator.getTerm(); + if (_query_normalization) { + Normalizing normalization = _query_normalization->normalizing_mode(iterator.getIndexName()); + TermType termType = ParseItem::toTermType(iterator.getType()); + v->visitKeyword(&item, QueryNormalization::optional_fold(term, termType, normalization), + prefix_like, isSpecialToken); + } else { + v->visitKeyword(&item, term, prefix_like, isSpecialToken); + } } break; case search::ParseItem::ITEM_NUMTERM: @@ -96,13 +110,6 @@ JuniperQueryAdapter::Traverse(juniper::IQueryVisitor *v) const if (!v->VisitPHRASE(&item, iterator.getArity())) rc = skipItem(&iterator); break; - case search::ParseItem::ITEM_PREFIXTERM: - case search::ParseItem::ITEM_SUBSTRINGTERM: - { - vespalib::stringref term = iterator.getTerm(); - v->visitKeyword(&item, term, true, isSpecialToken); - } - break; case search::ParseItem::ITEM_ANY: if (!v->VisitANY(&item, iterator.getArity())) rc = skipItem(&iterator); diff --git a/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.h b/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.h index 2d67ff77db9..a544c674aa9 100644 --- a/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.h +++ b/searchsummary/src/vespa/searchsummary/docsummary/juniper_query_adapter.h @@ -5,7 +5,10 @@ #include <vespa/juniper/query.h> #include <vespa/vespalib/stllike/string.h> -namespace search { class SimpleQueryStackDumpIterator; } +namespace search { + class SimpleQueryStackDumpIterator; + class QueryNormalization; +} namespace search::fef { class Properties; } namespace search::docsummary { @@ -19,6 +22,7 @@ class IQueryTermFilter; class JuniperQueryAdapter : public juniper::IQuery { private: + const QueryNormalization * _query_normalization; const IQueryTermFilter *_query_term_filter; const vespalib::stringref _buf; const search::fef::Properties & _highlightTerms; @@ -26,7 +30,7 @@ private: public: JuniperQueryAdapter(const JuniperQueryAdapter&) = delete; JuniperQueryAdapter operator= (const JuniperQueryAdapter&) = delete; - JuniperQueryAdapter(const IQueryTermFilter *query_term_filter, vespalib::stringref buf, + JuniperQueryAdapter(const QueryNormalization * normalization, const IQueryTermFilter *query_term_filter, vespalib::stringref buf, const search::fef::Properties & highlightTerms); ~JuniperQueryAdapter() override; bool skipItem(search::SimpleQueryStackDumpIterator *iterator) const; |