diff options
author | Tor Egge <Tor.Egge@online.no> | 2023-11-24 10:23:03 +0100 |
---|---|---|
committer | Tor Egge <Tor.Egge@online.no> | 2023-11-24 10:23:03 +0100 |
commit | 580dc0762fceb32a6c16678745b022ade9cf4146 (patch) | |
tree | 3825094440a43408e0622f656f006e3463aff68f /searchlib | |
parent | 7e1fc00d718e3c80f837d50da4e48cadf146987c (diff) |
Add InTerm to backend.
Diffstat (limited to 'searchlib')
19 files changed, 277 insertions, 17 deletions
diff --git a/searchlib/src/tests/query/customtypevisitor_test.cpp b/searchlib/src/tests/query/customtypevisitor_test.cpp index 9d8e60cb551..815ff3bf1fb 100644 --- a/searchlib/src/tests/query/customtypevisitor_test.cpp +++ b/searchlib/src/tests/query/customtypevisitor_test.cpp @@ -3,6 +3,7 @@ #include <vespa/searchlib/query/tree/customtypevisitor.h> #include <vespa/searchlib/query/tree/intermediatenodes.h> +#include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/testkit/testapp.h> @@ -48,6 +49,12 @@ struct MyNearestNeighborTerm : NearestNeighborTerm { }; struct MyTrue : TrueQueryNode {}; struct MyFalse : FalseQueryNode {}; +struct MyInTerm : InTerm { + MyInTerm() + : InTerm(std::make_unique<StringTermVector>(0), MultiTerm::Type::STRING, "view", 0, Weight(0)) + { + } +}; struct MyQueryNodeTypes { using And = MyAnd; @@ -76,6 +83,7 @@ struct MyQueryNodeTypes { using NearestNeighborTerm = MyNearestNeighborTerm; using FalseQueryNode = MyFalse; using TrueQueryNode = MyTrue; + using InTerm = MyInTerm; }; class MyCustomVisitor : public CustomTypeVisitor<MyQueryNodeTypes> @@ -115,6 +123,7 @@ public: void visit(MyTrue &) override { setVisited<MyTrue>(); } void visit(MyFalse &) override { setVisited<MyFalse>(); } void visit(MyFuzzyTerm &) override { setVisited<MyFuzzyTerm>(); } + void visit(MyInTerm&) override { setVisited<MyInTerm>(); } }; template <class T> diff --git a/searchlib/src/tests/query/query_visitor_test.cpp b/searchlib/src/tests/query/query_visitor_test.cpp index 00126544d78..b0c9c4e570e 100644 --- a/searchlib/src/tests/query/query_visitor_test.cpp +++ b/searchlib/src/tests/query/query_visitor_test.cpp @@ -5,6 +5,7 @@ #include <vespa/searchlib/query/tree/point.h> #include <vespa/searchlib/query/tree/queryvisitor.h> #include <vespa/searchlib/query/tree/simplequery.h> +#include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/searchlib/query/tree/termnodes.h> #include <vespa/vespalib/testkit/testapp.h> @@ -50,6 +51,7 @@ public: void visit(TrueQueryNode &) override { isVisited<TrueQueryNode>() = true; } void visit(FalseQueryNode &) override { isVisited<FalseQueryNode>() = true; } void visit(FuzzyTerm &) override { isVisited<FuzzyTerm>() = true; } + void visit(InTerm&) override { isVisited<InTerm>() = true; } }; template <class T> @@ -87,6 +89,7 @@ TEST("requireThatAllNodesCanBeVisited") { checkVisit<TrueQueryNode>(new SimpleTrue()); checkVisit<FalseQueryNode>(new SimpleFalse()); checkVisit<FuzzyTerm>(new SimpleFuzzyTerm("t", "field", 0, Weight(0), 2, 0)); + checkVisit<InTerm>(new SimpleInTerm(std::make_unique<StringTermVector>(0), MultiTerm::Type::STRING, "field", 0, Weight(0))); } } // namespace diff --git a/searchlib/src/tests/query/querybuilder_test.cpp b/searchlib/src/tests/query/querybuilder_test.cpp index 40af60d4a04..116496fbc70 100644 --- a/searchlib/src/tests/query/querybuilder_test.cpp +++ b/searchlib/src/tests/query/querybuilder_test.cpp @@ -456,6 +456,13 @@ struct MyFuzzyTerm : FuzzyTerm { : FuzzyTerm(t, f, i, w, m, p) { } }; +struct MyInTerm : InTerm { + MyInTerm(std::unique_ptr<TermVector> terms, MultiTerm::Type type, + const string& f, int32_t i, Weight w) + : InTerm(std::move(terms), type, f, i, w) + { + } +}; struct MyQueryNodeTypes { using And = MyAnd; @@ -484,6 +491,7 @@ struct MyQueryNodeTypes { using TrueQueryNode = MyTrue; using FalseQueryNode = MyFalse; using FuzzyTerm = MyFuzzyTerm; + using InTerm = MyInTerm; }; TEST("require that Custom Query Trees Can Be Built") { @@ -673,7 +681,7 @@ TEST("require that empty intermediate node can be added") { } TEST("control size of SimpleQueryStackDumpIterator") { - EXPECT_EQUAL(120u, sizeof(SimpleQueryStackDumpIterator)); + EXPECT_EQUAL(128u, sizeof(SimpleQueryStackDumpIterator)); } TEST("test query parsing error") { @@ -785,6 +793,74 @@ TEST("first integer then string MultiTerm") { verify_multiterm_get(mt); } +namespace { + +std::vector<vespalib::string> in_strings = { "this", "is", "a", "test" }; + +std::vector<int64_t> in_integers = { 24, INT64_C(93000000000) }; + +template <typename TermType> +std::unique_ptr<TermVector> +make_subterms(const std::vector<TermType>& values) +{ + using TermVectorType = std::conditional_t<std::is_same_v<TermType,int64_t>,IntegerTermVector,StringTermVector>; + auto terms = std::make_unique<TermVectorType>(in_strings.size()); + for (auto term : values) { + terms->addTerm(term); + } + return terms; +} + +template <typename TermType> +void +verify_subterms(InTerm& in_term, const std::vector<TermType>& values) +{ + EXPECT_EQUAL(values.size(), in_term.getNumTerms()); + uint32_t i = 0; + for (auto term : values) { + if constexpr (std::is_same_v<TermType, int64_t>) { + EXPECT_EQUAL(term, in_term.getAsInteger(i).first); + } else { + EXPECT_EQUAL(term, in_term.getAsString(i).first); + } + ++i; + } +} + +template <typename TermType> +void +verify_in_node(Node& node, const std::vector<TermType>& values) +{ + auto in_term = as_node<InTerm>(&node); + verify_subterms(*in_term, values); +} + +template <typename TermType> +void +test_in_node(const std::vector<TermType>& values) +{ + QueryBuilder<SimpleQueryNodeTypes> builder; + builder.add_in_term(make_subterms(values), MultiTerm::Type::STRING, + "view", 0, Weight(0)); + auto node = builder.build(); + string stack_dump = StackDumpCreator::create(*node); + SimpleQueryStackDumpIterator iterator(stack_dump); + verify_in_node(*QueryTreeCreator<SimpleQueryNodeTypes>::create(iterator), values); + verify_in_node(*QueryTreeCreator<SimpleQueryNodeTypes>::replicate(*node), values); +} + +} + +TEST("require that in_term with strings can be created") +{ + test_in_node(in_strings); +} + +TEST("require that in_term with integers can be created") +{ + test_in_node(in_integers); +} + } // namespace TEST_MAIN() { TEST_RUN_ALL(); } diff --git a/searchlib/src/vespa/searchlib/parsequery/parse.h b/searchlib/src/vespa/searchlib/parsequery/parse.h index 7029a3d4e12..89996515a4a 100644 --- a/searchlib/src/vespa/searchlib/parsequery/parse.h +++ b/searchlib/src/vespa/searchlib/parsequery/parse.h @@ -57,11 +57,21 @@ public: ITEM_TRUE = 28, ITEM_FALSE = 29, ITEM_FUZZY = 30, - ITEM_UNDEF = 32, - // NOTE: Only 5 bits are used to encode the item type in the protocol, and 31 is the last available value. - // We might need to use 31 to signal a protocol extension in order to support more item types. + ITEM_STRING_IN = 31, + ITEM_NUMERIC_IN = 32, + ITEM_UNDEF = 33, }; + /* + * Mask for item type. 5 bits item type, 3 bits item features. + */ + static constexpr uint8_t item_type_mask = 31; + /* + * Value encoded as item type in original serialization to indicate + * that an additional byte is needed for item type. + */ + static constexpr uint8_t item_type_extension_mark = 31; + /** A tag identifying the origin of this query node. */ using ItemCreator = parseitem::ItemCreator; @@ -78,10 +88,8 @@ public: IFLAG_NOPOSITIONDATA = 0x00000004, // we should not use position data when ranking this term }; - /** Extra information on each item (creator id) coded in bits 12-19 of _type */ - static inline ItemCreator GetCreator(uint8_t type) { return static_cast<ItemCreator>((type >> 3) & 0x01); } - /** The old item type now uses only the lower 12 bits in a backward compatible way) */ - static inline ItemType GetType(uint8_t type) { return static_cast<ItemType>(type & 0x1F); } + /** Extra information on each item (creator id) coded in bit 3 of flags */ + static inline ItemCreator GetCreator(uint8_t flags) { return static_cast<ItemCreator>((flags >> 3) & 0x01); } static inline bool GetFeature(uint8_t type, uint8_t feature) { return ((type & feature) != 0); } diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp index ac54ea205be..596ad53c014 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.cpp @@ -1,12 +1,16 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. #include "stackdumpiterator.h" +#include <vespa/searchlib/query/tree/integer_term_vector.h> #include <vespa/searchlib/query/tree/predicate_query_term.h> +#include <vespa/searchlib/query/tree/string_term_vector.h> #include <vespa/vespalib/util/compress.h> #include <vespa/vespalib/objects/nbo.h> #include <cassert> +using search::query::IntegerTermVector; using search::query::PredicateQueryTerm; +using search::query::StringTermVector; namespace search { @@ -28,7 +32,8 @@ SimpleQueryStackDumpIterator::SimpleQueryStackDumpIterator(vespalib::stringref b _extraIntArg3(0), _extraDoubleArg4(0), _extraDoubleArg5(0), - _predicate_query_term() + _predicate_query_term(), + _terms() { } @@ -101,7 +106,14 @@ bool SimpleQueryStackDumpIterator::readNext() { // Find an item at the current position const char *p = _buf + _currPos; uint8_t typefield = *p++; - _currType = ParseItem::GetType(typefield); + uint8_t type_code = typefield & ParseItem::item_type_mask; + if (type_code == ParseItem::item_type_extension_mark) { + if (p >= _bufEnd || ((uint8_t) *p) >= 0x80) { + return false; + } + type_code += (uint8_t) *p++; + } + _currType = static_cast<ParseItem::ItemType>(type_code); if (ParseItem::GetFeature_Weight(typefield)) { int64_t tmpLong = readCompressedInt(p); @@ -199,6 +211,12 @@ bool SimpleQueryStackDumpIterator::readNext() { // no content _currArity = 0; break; + case ParseItem::ITEM_STRING_IN: + read_string_in(p); + break; + case ParseItem::ITEM_NUMERIC_IN: + read_numeric_in(p); + break; default: // Unknown item, so report that no more are available return false; @@ -271,4 +289,38 @@ SimpleQueryStackDumpIterator::getPredicateQueryTerm() return std::move(_predicate_query_term); } +void +SimpleQueryStackDumpIterator::read_string_in(const char*& p) +{ + uint32_t num_terms = readCompressedPositiveInt(p); + _currArity = 0; + _curr_index_name = read_stringref(p); + _curr_term = vespalib::stringref(); + auto terms = std::make_unique<StringTermVector>(num_terms); + for (uint32_t i = 0; i < num_terms; ++i) { + terms->addTerm(read_stringref(p)); + } + _terms = std::move(terms); +} + +void +SimpleQueryStackDumpIterator::read_numeric_in(const char*& p) +{ + uint32_t num_terms = readCompressedPositiveInt(p); + _currArity = 0; + _curr_index_name = read_stringref(p); + _curr_term = vespalib::stringref(); + auto terms = std::make_unique<IntegerTermVector>(num_terms); + for (uint32_t i = 0; i < num_terms; ++i) { + terms->addTerm(read_value<int64_t>(p)); + } + _terms = std::move(terms); +} + +std::unique_ptr<query::TermVector> +SimpleQueryStackDumpIterator::get_terms() +{ + return std::move(_terms); +} + } diff --git a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h index add26ac7938..e04f6379434 100644 --- a/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h +++ b/searchlib/src/vespa/searchlib/parsequery/stackdumpiterator.h @@ -6,7 +6,12 @@ #include <vespa/vespalib/stllike/string.h> #include <memory> -namespace search::query { class PredicateQueryTerm; } +namespace search::query { + +class PredicateQueryTerm; +class TermVector; + +} namespace search { /** @@ -48,6 +53,7 @@ private: double _extraDoubleArg5; /** The predicate query specification */ std::unique_ptr<query::PredicateQueryTerm> _predicate_query_term; + std::unique_ptr<query::TermVector> _terms; VESPA_DLL_LOCAL vespalib::stringref read_stringref(const char *&p); VESPA_DLL_LOCAL uint64_t readCompressedPositiveInt(const char *&p); @@ -58,6 +64,8 @@ private: VESPA_DLL_LOCAL void readNN(const char *&p); VESPA_DLL_LOCAL void readComplexTerm(const char *& p); VESPA_DLL_LOCAL void readFuzzy(const char *&p); + VESPA_DLL_LOCAL void read_string_in(const char*& p); + VESPA_DLL_LOCAL void read_numeric_in(const char*& p); VESPA_DLL_LOCAL bool readNext(); public: /** @@ -124,6 +132,7 @@ public: uint32_t getFuzzyPrefixLength() const { return _extraIntArg2; } std::unique_ptr<query::PredicateQueryTerm> getPredicateQueryTerm(); + std::unique_ptr<query::TermVector> get_terms(); vespalib::stringref getIndexName() const { return _curr_index_name; } vespalib::stringref getTerm() const { return _curr_term; } diff --git a/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h b/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h index 760c21983f1..43396f33b8e 100644 --- a/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/customtypevisitor.h @@ -53,6 +53,7 @@ public: virtual void visit(typename NodeTypes::TrueQueryNode &) = 0; virtual void visit(typename NodeTypes::FalseQueryNode &) = 0; virtual void visit(typename NodeTypes::FuzzyTerm &) = 0; + virtual void visit(typename NodeTypes::InTerm&) = 0; private: // Route QueryVisit requests to the correct custom type. @@ -83,6 +84,7 @@ private: using TTrueQueryNode = typename NodeTypes::TrueQueryNode; using TFalseQueryNode = typename NodeTypes::FalseQueryNode; using TFuzzyTerm = typename NodeTypes::FuzzyTerm; + using TInTerm = typename NodeTypes::InTerm; void visit(And &n) override { visit(static_cast<TAnd&>(n)); } void visit(AndNot &n) override { visit(static_cast<TAndNot&>(n)); } @@ -110,6 +112,7 @@ private: void visit(TrueQueryNode &n) override { visit(static_cast<TTrueQueryNode&>(n)); } void visit(FalseQueryNode &n) override { visit(static_cast<TFalseQueryNode&>(n)); } void visit(FuzzyTerm &n) override { visit(static_cast<TFuzzyTerm &>(n)); } + void visit(InTerm& n) override { visit(static_cast<TInTerm&>(n)); } }; } diff --git a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h index 4f9632e8df9..a8db09bb03f 100644 --- a/searchlib/src/vespa/searchlib/query/tree/querybuilder.h +++ b/searchlib/src/vespa/searchlib/query/tree/querybuilder.h @@ -21,6 +21,7 @@ #include "predicate_query_term.h" #include "node.h" +#include "termnodes.h" #include "const_bool_nodes.h" #include <vespa/searchlib/query/weight.h> #include <stack> @@ -30,6 +31,7 @@ namespace search::query { class Intermediate; class Location; class Range; +class TermVector; class QueryBuilderBase { @@ -220,6 +222,7 @@ create_nearest_neighbor_term(vespalib::stringref query_tensor_name, vespalib::st target_num_hits, allow_approximate, explore_additional_hits, distance_threshold); } + template <class NodeTypes> typename NodeTypes::FuzzyTerm * createFuzzyTerm(vespalib::stringref term, vespalib::stringref view, int32_t id, Weight weight, @@ -227,6 +230,11 @@ createFuzzyTerm(vespalib::stringref term, vespalib::stringref view, int32_t id, return new typename NodeTypes::FuzzyTerm(term, view, id, weight, maxEditDistance, prefixLength); } +template <class NodeTypes> +typename NodeTypes::InTerm * +create_in_term(std::unique_ptr<TermVector> terms, MultiTerm::Type type, vespalib::stringref view, int32_t id, Weight weight) { + return new typename NodeTypes::InTerm(std::move(terms), type, view, id, weight); +} template <class NodeTypes> class QueryBuilder : public QueryBuilderBase { @@ -353,6 +361,10 @@ public: typename NodeTypes::FalseQueryNode &add_false_node() { return addTerm(create_false<NodeTypes>()); } + typename NodeTypes::InTerm& add_in_term(std::unique_ptr<TermVector> terms, MultiTerm::Type type, stringref view, int32_t id, Weight weight) { + adjustWeight(weight); + return addTerm(create_in_term<NodeTypes>(std::move(terms), type, view, id, weight)); + } }; } diff --git a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h index b7cce0f15b1..f578bb57e21 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryreplicator.h @@ -2,9 +2,11 @@ #pragma once +#include "integer_term_vector.h" #include "intermediatenodes.h" #include "querybuilder.h" #include "queryvisitor.h" +#include "string_term_vector.h" #include "termnodes.h" namespace search::query { @@ -201,6 +203,30 @@ private: node.getId(), node.getWeight(), node.getMaxEditDistance(), node.getPrefixLength())); } + + std::unique_ptr<TermVector> replicate_subterms(const InTerm& original) { + uint32_t num_terms = original.getNumTerms(); + if (original.getType() == MultiTerm::Type::STRING) { + auto replica = std::make_unique<StringTermVector>(num_terms); + for (uint32_t i(0); i < num_terms; i++) { + auto v = original.getAsString(i); + replica->addTerm(v.first); + } + return replica; + } else if (original.getType() == MultiTerm::Type::INTEGER) { + auto replica = std::make_unique<IntegerTermVector>(num_terms); + for (uint32_t i(0); i < original.getNumTerms(); i++) { + auto v = original.getAsInteger(i); + replica->addTerm(v.first); + } + return replica; + } else { + abort(); + } + } + void visit(InTerm& node) override { + replicate(node, _builder.add_in_term(replicate_subterms(node), node.getType(), node.getView(), node.getId(), node.getWeight())); + } }; } diff --git a/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h b/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h index 1b24a524abb..eb089fb81c3 100644 --- a/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/queryvisitor.h @@ -30,6 +30,7 @@ class NearestNeighborTerm; class TrueQueryNode; class FalseQueryNode; class FuzzyTerm; +class InTerm; struct QueryVisitor { virtual ~QueryVisitor() {} @@ -60,6 +61,7 @@ struct QueryVisitor { virtual void visit(TrueQueryNode &) = 0; virtual void visit(FalseQueryNode &) = 0; virtual void visit(FuzzyTerm &) = 0; + virtual void visit(InTerm &) = 0; }; } diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.cpp b/searchlib/src/vespa/searchlib/query/tree/simplequery.cpp index d1210bf6c0b..4cfc1d939c3 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.cpp @@ -54,4 +54,6 @@ SimpleNearestNeighborTerm::~SimpleNearestNeighborTerm() = default; SimpleFuzzyTerm::~SimpleFuzzyTerm() = default; +SimpleInTerm::~SimpleInTerm() = default; + } diff --git a/searchlib/src/vespa/searchlib/query/tree/simplequery.h b/searchlib/src/vespa/searchlib/query/tree/simplequery.h index 51af979a230..535402bf14d 100644 --- a/searchlib/src/vespa/searchlib/query/tree/simplequery.h +++ b/searchlib/src/vespa/searchlib/query/tree/simplequery.h @@ -74,6 +74,13 @@ struct SimpleWandTerm : WandTerm { : WandTerm(num_terms, view, id, weight, targetNumHits, scoreThreshold, thresholdBoostFactor) {} ~SimpleWandTerm() override; }; +struct SimpleInTerm : InTerm { + SimpleInTerm(std::unique_ptr<TermVector> terms, MultiTerm::Type type, vespalib::stringref view, int32_t id, Weight weight) + : InTerm(std::move(terms), type, view, id, weight) + { + } + ~SimpleInTerm() override; +}; struct SimpleRank : Rank { ~SimpleRank() override; @@ -189,6 +196,7 @@ struct SimpleQueryNodeTypes { using RegExpTerm = SimpleRegExpTerm; using NearestNeighborTerm = SimpleNearestNeighborTerm; using FuzzyTerm = SimpleFuzzyTerm; + using InTerm = SimpleInTerm; }; } diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp index 465263afeb4..6707712ee69 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpcreator.cpp @@ -52,10 +52,14 @@ class QueryNodeConverter : public QueryVisitor { } void append_type_and_features(ParseItem::ItemType type, uint8_t item_features) { - assert(static_cast<uint32_t>(type) < 31u); - uint8_t type_and_features = (static_cast<uint8_t>(type) | item_features); - _buf.preAlloc(sizeof(uint8_t)); - _buf.append(&type_and_features, sizeof(uint8_t)); + uint32_t type_code = static_cast<uint32_t>(type); + assert(type_code < ParseItem::item_type_extension_mark + 0x80); + if (type_code >= ParseItem::item_type_extension_mark) { + appendByte(ParseItem::item_type_extension_mark | item_features); + appendByte(type - ParseItem::item_type_extension_mark); + } else { + appendByte(type_code | item_features); + } } void appendDouble(double i) { @@ -169,7 +173,9 @@ class QueryNodeConverter : public QueryVisitor { features |= ParseItem::IF_FLAGS; } append_type_and_features(type, features); - appendCompressedNumber(node.getWeight().percent()); + if ((features & ParseItem::IF_WEIGHT) != 0) { + appendCompressedNumber(node.getWeight().percent()); + } if ((features & ParseItem::IF_FLAGS) != 0) { appendByte(flags); } @@ -296,6 +302,22 @@ class QueryNodeConverter : public QueryVisitor { appendDouble(node.get_distance_threshold()); } + void visit(InTerm& node) override { + bool is_string = (node.getType() == MultiTerm::Type::STRING); + auto item_type = is_string ? ParseItem::ITEM_STRING_IN : ParseItem::ITEM_NUMERIC_IN; + createWeightedSet(node, item_type, 0); + auto num_terms = node.getNumTerms(); + if (is_string) { + for (size_t i = 0; i < num_terms; ++i) { + appendString(node.getAsString(i).first); + } + } else { + for (size_t i = 0; i < num_terms; ++i) { + appendLong(node.getAsInteger(i).first); + } + } + } + public: QueryNodeConverter() : _buf(4_Ki) diff --git a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h index c593bba48b1..7c743a72567 100644 --- a/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h +++ b/searchlib/src/vespa/searchlib/query/tree/stackdumpquerycreator.h @@ -188,6 +188,10 @@ private: uint32_t maxEditDistance = queryStack.getFuzzyMaxEditDistance(); uint32_t prefixLength = queryStack.getFuzzyPrefixLength(); t = &builder.addFuzzyTerm(term, view, id, weight, maxEditDistance, prefixLength); + } else if (type == ParseItem::ITEM_STRING_IN) { + t = &builder.add_in_term(queryStack.get_terms(), MultiTerm::Type::STRING, view, id, weight); + } else if (type == ParseItem::ITEM_NUMERIC_IN) { + t = &builder.add_in_term(queryStack.get_terms(), MultiTerm::Type::INTEGER, view, id, weight); } else { vespalib::Issue::report("query builder: Unable to create query tree from stack dump. node type = %d.", type); } diff --git a/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h b/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h index 3bb6f93cb00..9d94e12cfcd 100644 --- a/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h +++ b/searchlib/src/vespa/searchlib/query/tree/templatetermvisitor.h @@ -58,6 +58,8 @@ class TemplateTermVisitor : public CustomTypeTermVisitor<NodeTypes> { // term's children, unless this member function is overridden // to do so. void visit(typename NodeTypes::SameElement &n) override { myVisit(n); } + + void visit(typename NodeTypes::InTerm& n) override { myVisit(n); } }; } diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp index a8fbe81a222..4adfc4f9444 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.cpp @@ -26,6 +26,7 @@ WeightedSetTerm::~WeightedSetTerm() = default; DotProduct::~DotProduct() = default; WandTerm::~WandTerm() = default; FuzzyTerm::~FuzzyTerm() = default; +InTerm::~InTerm() = default; namespace { @@ -97,6 +98,13 @@ MultiTerm::MultiTerm(uint32_t num_terms) _type(Type::UNKNOWN) {} +MultiTerm::MultiTerm(std::unique_ptr<TermVector> terms, Type type) + : _terms(std::move(terms)), + _num_terms(_terms->size()), + _type(type) +{ +} + MultiTerm::~MultiTerm() = default; std::unique_ptr<TermVector> diff --git a/searchlib/src/vespa/searchlib/query/tree/termnodes.h b/searchlib/src/vespa/searchlib/query/tree/termnodes.h index 5db11bf6ba9..97e659dcc77 100644 --- a/searchlib/src/vespa/searchlib/query/tree/termnodes.h +++ b/searchlib/src/vespa/searchlib/query/tree/termnodes.h @@ -191,7 +191,8 @@ public: uint32_t getNumTerms() const { return _num_terms; } Type getType() const { return _type; } protected: - MultiTerm(uint32_t num_terms); + explicit MultiTerm(uint32_t num_terms); + MultiTerm(std::unique_ptr<TermVector> terms, Type type); private: VESPA_DLL_LOCAL std::unique_ptr<TermVector> downgrade() __attribute__((noinline)); std::unique_ptr<TermVector> _terms; @@ -237,4 +238,14 @@ public: double getThresholdBoostFactor() const { return _thresholdBoostFactor; } }; +class InTerm : public QueryNodeMixin<InTerm, MultiTerm>, public Term { +public: + InTerm(std::unique_ptr<TermVector> terms, MultiTerm::Type type, const vespalib::string& view, int32_t id, Weight weight) + : QueryNodeMixinType(std::move(terms), type), + Term(view, id, weight) + { + } + virtual ~InTerm() = 0; +}; + } diff --git a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h index c054f43e137..26d6f1d384d 100644 --- a/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h +++ b/searchlib/src/vespa/searchlib/queryeval/create_blueprint_visitor_helper.h @@ -69,6 +69,7 @@ public: void visit(query::WeightedSetTerm &n) override { visitWeightedSetTerm(n); } void visit(query::DotProduct &n) override { visitDotProduct(n); } void visit(query::WandTerm &n) override { visitWandTerm(n); } + void visit(query::InTerm&) override { illegalVisit(); } void visit(query::NumberTerm &n) override = 0; void visit(query::LocationTerm &n) override = 0; diff --git a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp index 52e6e4dbf8c..d7845d9ef28 100644 --- a/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp +++ b/searchlib/src/vespa/searchlib/queryeval/termasstring.cpp @@ -18,6 +18,7 @@ using search::query::DotProduct; using search::query::Equiv; using search::query::FalseQueryNode; using search::query::FuzzyTerm; +using search::query::InTerm; using search::query::LocationTerm; using search::query::Near; using search::query::NearestNeighborTerm; @@ -97,6 +98,7 @@ struct TermAsStringVisitor : public QueryVisitor { void visit(WeightedSetTerm &) override {illegalVisit(); } void visit(DotProduct &) override {illegalVisit(); } void visit(WandTerm &) override {illegalVisit(); } + void visit(InTerm&) override { illegalVisit(); } void visit(NumberTerm &n) override {visitTerm(n); } void visit(LocationTerm &n) override {visitTerm(n); } |