diff options
author | Alexey Chernyshev <aleksei@spotify.com> | 2022-03-18 11:48:03 +0100 |
---|---|---|
committer | Alexey Chernyshev <aleksei@spotify.com> | 2022-03-23 16:21:04 +0100 |
commit | 984c1acbfe71589aba7ab14dab00359f009fed9e (patch) | |
tree | dec3705741f6771cd00ed34a1d965b3a22e58f34 /container-search | |
parent | d9805209e3b0e33be3c0cc454c4604043663c1c4 (diff) |
Addressing comments + fixing test
Diffstat (limited to 'container-search')
6 files changed, 261 insertions, 22 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java index 9a710deca67..74c31a5d1f0 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java @@ -9,16 +9,16 @@ import java.nio.ByteBuffer; * @author alexeyche */ public class FuzzyItem extends TermItem { - private String fuzzyQuery; + private String term; - public FuzzyItem(String indexName, boolean isFromQuery, String fuzzyQuery) { + public FuzzyItem(String indexName, boolean isFromQuery, String term) { super(indexName, isFromQuery, null); - setValue(fuzzyQuery); + setValue(term); } @Override public void setValue(String value) { - this.fuzzyQuery = value; + this.term = value; } @Override @@ -43,7 +43,7 @@ public class FuzzyItem extends TermItem { @Override public String stringValue() { - return fuzzyQuery; + return term; } @Override @@ -73,11 +73,11 @@ public class FuzzyItem extends TermItem { return false; } FuzzyItem other = (FuzzyItem) obj; - if (fuzzyQuery == null) { - if (other.fuzzyQuery != null) { + if (term == null) { + if (other.term != null) { return false; } - } else if (!fuzzyQuery.equals(other.fuzzyQuery)) { + } else if (!term.equals(other.term)) { return false; } return true; @@ -87,17 +87,18 @@ public class FuzzyItem extends TermItem { public int hashCode() { final int prime = 31; int result = super.hashCode(); - result = prime * result + ((fuzzyQuery == null) ? 0 : fuzzyQuery.hashCode()); + result = prime * result + ((term == null) ? 0 : term.hashCode()); return result; } @Override public String toString() { StringBuilder builder = new StringBuilder(); - builder.append("FuzzyItem [fuzzyQuery=").append(fuzzyQuery).append("]"); + builder.append("FuzzyItem [term=").append(term).append("]"); return builder.toString(); } + @Override protected void encodeThis(ByteBuffer buffer) { super.encodeThis(buffer); putString(getIndexedString(), buffer); diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 6f7aff798a5..320148ec01d 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -2,7 +2,6 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; -import com.yahoo.prelude.query.*; import com.yahoo.processing.IllegalInputException; import com.yahoo.collections.LazyMap; import com.yahoo.geo.DistanceParser; @@ -11,6 +10,39 @@ import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.BoolItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.FuzzyItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NearestNeighborItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.ONearItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RangeItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.Parser; @@ -29,13 +61,66 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import static com.yahoo.search.yql.YqlParser.*; import static com.yahoo.slime.Type.ARRAY; import static com.yahoo.slime.Type.DOUBLE; import static com.yahoo.slime.Type.LONG; import static com.yahoo.slime.Type.OBJECT; import static com.yahoo.slime.Type.STRING; +import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; +import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; +import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; +import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; +import static com.yahoo.search.yql.YqlParser.APPROXIMATE; +import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; +import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; +import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; +import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; +import static com.yahoo.search.yql.YqlParser.DISTANCE; +import static com.yahoo.search.yql.YqlParser.DISTANCE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; +import static com.yahoo.search.yql.YqlParser.EQUIV; +import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.FUZZY; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; +import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; +import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; +import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; +import static com.yahoo.search.yql.YqlParser.LABEL; +import static com.yahoo.search.yql.YqlParser.NEAR; +import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; +import static com.yahoo.search.yql.YqlParser.NFKC; +import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; +import static com.yahoo.search.yql.YqlParser.ONEAR; +import static com.yahoo.search.yql.YqlParser.ORIGIN; +import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; +import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; +import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; +import static com.yahoo.search.yql.YqlParser.PHRASE; +import static com.yahoo.search.yql.YqlParser.PREDICATE; +import static com.yahoo.search.yql.YqlParser.PREFIX; +import static com.yahoo.search.yql.YqlParser.RANGE; +import static com.yahoo.search.yql.YqlParser.RANK; +import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; +import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; +import static com.yahoo.search.yql.YqlParser.STEM; +import static com.yahoo.search.yql.YqlParser.SUBSTRING; +import static com.yahoo.search.yql.YqlParser.SUFFIX; +import static com.yahoo.search.yql.YqlParser.TARGET_HITS; +import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; +import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; +import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; +import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; +import static com.yahoo.search.yql.YqlParser.WAND; +import static com.yahoo.search.yql.YqlParser.WEAK_AND; +import static com.yahoo.search.yql.YqlParser.WEIGHT; +import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; + /** * The Select query language. * diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index afc903ba7cf..e778798b0e5 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -1,6 +1,56 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; +import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; +import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; +import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; +import static com.yahoo.search.yql.YqlParser.BOUNDS; +import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN; +import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN; +import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; +import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; +import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; +import static com.yahoo.search.yql.YqlParser.DISTANCE; +import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; +import static com.yahoo.search.yql.YqlParser.END_ANCHOR; +import static com.yahoo.search.yql.YqlParser.EQUIV; +import static com.yahoo.search.yql.YqlParser.FILTER; +import static com.yahoo.search.yql.YqlParser.FUZZY; +import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; +import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; +import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; +import static com.yahoo.search.yql.YqlParser.LABEL; +import static com.yahoo.search.yql.YqlParser.NEAR; +import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; +import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; +import static com.yahoo.search.yql.YqlParser.ONEAR; +import static com.yahoo.search.yql.YqlParser.ORIGIN; +import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; +import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; +import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; +import static com.yahoo.search.yql.YqlParser.PHRASE; +import static com.yahoo.search.yql.YqlParser.PREFIX; +import static com.yahoo.search.yql.YqlParser.RANGE; +import static com.yahoo.search.yql.YqlParser.RANK; +import static com.yahoo.search.yql.YqlParser.RANKED; +import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; +import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; +import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; +import static com.yahoo.search.yql.YqlParser.START_ANCHOR; +import static com.yahoo.search.yql.YqlParser.STEM; +import static com.yahoo.search.yql.YqlParser.SUBSTRING; +import static com.yahoo.search.yql.YqlParser.SUFFIX; +import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; +import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; +import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; +import static com.yahoo.search.yql.YqlParser.URI; +import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; +import static com.yahoo.search.yql.YqlParser.WAND; +import static com.yahoo.search.yql.YqlParser.WEAK_AND; +import static com.yahoo.search.yql.YqlParser.WEIGHT; +import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; + import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; @@ -15,15 +65,52 @@ import java.util.Map; import java.util.Map.Entry; import com.google.common.collect.ImmutableMap; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BoolItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.FalseItem; +import com.yahoo.prelude.query.FuzzyItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.GeoLocationItem; +import com.yahoo.prelude.query.MarkerWordItem; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NearestNeighborItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.ONearItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RangeItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.ToolBox; import com.yahoo.prelude.query.ToolBox.QueryVisitor; +import com.yahoo.prelude.query.TrueItem; +import com.yahoo.prelude.query.UriItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.query.QueryTree; -import static com.yahoo.search.yql.YqlParser.*; - /** * Serialize Vespa query trees to YQL+ strings. * diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 99a449026d8..06ee0e706f3 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -27,8 +27,49 @@ import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.AndSegmentItem; +import com.yahoo.prelude.query.BoolItem; +import com.yahoo.prelude.query.CompositeItem; +import com.yahoo.prelude.query.DotProductItem; +import com.yahoo.prelude.query.EquivItem; +import com.yahoo.prelude.query.FalseItem; +import com.yahoo.prelude.query.FuzzyItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.IntItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.Limit; +import com.yahoo.prelude.query.GeoLocationItem; +import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NearestNeighborItem; +import com.yahoo.prelude.query.NotItem; +import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.ONearItem; +import com.yahoo.prelude.query.OrItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.PredicateQueryItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.RangeItem; +import com.yahoo.prelude.query.RankItem; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SameElementItem; +import com.yahoo.prelude.query.SegmentItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.TaggableItem; +import com.yahoo.prelude.query.TermItem; +import com.yahoo.prelude.query.ToolBox; import com.yahoo.prelude.query.ToolBox.QueryVisitor; +import com.yahoo.prelude.query.TrueItem; +import com.yahoo.prelude.query.UriItem; +import com.yahoo.prelude.query.WandItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WeightedSetItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; @@ -1132,7 +1173,7 @@ public class YqlParser implements Parser { assertHasOperator(ast, ExpressionOperator.CONTAINS); String field = getIndex(ast.getArgument(0)); if (userQuery != null && indexFactsSession.getIndex(field).isAttribute()) { - userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly (unless fuzzy query is used)", 2); + userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly (unless fuzzy is used)", 2); } return instantiateLeafItem(field, ast.<OperatorNode<ExpressionOperator>> getArgument(1)); } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index a057d6f7c16..f40e212adde 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -8,7 +8,25 @@ import com.yahoo.prelude.Index; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.IndexModel; import com.yahoo.prelude.SearchDefinition; -import com.yahoo.prelude.query.*; +import com.yahoo.prelude.query.AndItem; +import com.yahoo.prelude.query.BoolItem; +import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.FuzzyItem; +import com.yahoo.prelude.query.IndexedItem; +import com.yahoo.prelude.query.Item; +import com.yahoo.prelude.query.MarkerWordItem; +import com.yahoo.prelude.query.PhraseItem; +import com.yahoo.prelude.query.PhraseSegmentItem; +import com.yahoo.prelude.query.PrefixItem; +import com.yahoo.prelude.query.QueryCanonicalizer; +import com.yahoo.prelude.query.RegExpItem; +import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.Substring; +import com.yahoo.prelude.query.SubstringItem; +import com.yahoo.prelude.query.SuffixItem; +import com.yahoo.prelude.query.WeakAndItem; +import com.yahoo.prelude.query.WordAlternativesItem; +import com.yahoo.prelude.query.WordItem; import com.yahoo.prelude.querytransform.QueryRewrite; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; @@ -366,8 +384,11 @@ public class YqlParserTestCase { @Test public void testFuzzy() { - assertParse("select foo from bar where baz contains fuzzy(\"a\")", - "FUZZY baz:a"); + QueryTree x = parse("select foo from bar where baz contains fuzzy(\"a b\")"); + Item root = x.getRoot(); + assertSame(FuzzyItem.class, root.getClass()); + assertEquals("baz", ((FuzzyItem) root).getIndexName()); + assertEquals("a b", ((FuzzyItem) root).stringValue()); } @Test diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java index 1c8433541e1..9b867be1484 100644 --- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java +++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java @@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ArrayNode; import com.fasterxml.jackson.databind.node.ObjectNode; import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.ExactStringItem; +import com.yahoo.prelude.query.FuzzyItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PrefixItem; @@ -676,8 +677,11 @@ public class SelectTestCase { @Test public void testFuzzy() { - assertParse("{ \"contains\": [\"description\", { \"fuzzy\": [\"a\"] }] }", - "FUZZY description:a"); + QueryTree x = parseWhere("{ \"contains\": [\"description\", { \"fuzzy\": [\"a b\"] }] }"); + Item root = x.getRoot(); + assertSame(FuzzyItem.class, root.getClass()); + assertEquals("description", ((FuzzyItem) root).getIndexName()); + assertEquals("a b", ((FuzzyItem) root).stringValue()); } //------------------------------------------------------------------- grouping tests |