diff options
Diffstat (limited to 'container-search/src/main/java/com/yahoo')
5 files changed, 167 insertions, 217 deletions
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java new file mode 100644 index 00000000000..9a710deca67 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java @@ -0,0 +1,106 @@ +// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. +package com.yahoo.prelude.query; + +import java.nio.ByteBuffer; + +/** + * Fuzzy search term + * + * @author alexeyche + */ +public class FuzzyItem extends TermItem { + private String fuzzyQuery; + + public FuzzyItem(String indexName, boolean isFromQuery, String fuzzyQuery) { + super(indexName, isFromQuery, null); + setValue(fuzzyQuery); + } + + @Override + public void setValue(String value) { + this.fuzzyQuery = value; + } + + @Override + public String getRawWord() { + return stringValue(); + } + + @Override + public boolean isWords() { + return false; + } + + @Override + public ItemType getItemType() { + return ItemType.FUZZY; + } + + @Override + public String getName() { + return "FUZZY"; + } + + @Override + public String stringValue() { + return fuzzyQuery; + } + + @Override + public boolean isStemmed() { + return false; + } + + @Override + public String getIndexedString() { + return stringValue(); + } + + @Override + public int getNumWords() { + return 1; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!super.equals(obj)) { + return false; + } + if (getClass() != obj.getClass()) { + return false; + } + FuzzyItem other = (FuzzyItem) obj; + if (fuzzyQuery == null) { + if (other.fuzzyQuery != null) { + return false; + } + } else if (!fuzzyQuery.equals(other.fuzzyQuery)) { + return false; + } + return true; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((fuzzyQuery == null) ? 0 : fuzzyQuery.hashCode()); + return result; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("FuzzyItem [fuzzyQuery=").append(fuzzyQuery).append("]"); + return builder.toString(); + } + + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + putString(getIndexedString(), buffer); + } +} + diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index 2e0c3cf8593..02b208b6ce1 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -56,7 +56,8 @@ public abstract class Item implements Cloneable { NEAREST_NEIGHBOR(26), GEO_LOCATION_TERM(27), TRUE(28), - FALSE(29); + FALSE(29), + FUZZY(30); public final int code; diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java index 1805a11ff5e..6f7aff798a5 100644 --- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java +++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java @@ -2,6 +2,7 @@ package com.yahoo.search.query; import com.google.common.base.Preconditions; +import com.yahoo.prelude.query.*; import com.yahoo.processing.IllegalInputException; import com.yahoo.collections.LazyMap; import com.yahoo.geo.DistanceParser; @@ -10,38 +11,6 @@ import com.yahoo.language.Language; import com.yahoo.language.process.Normalizer; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.BoolItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.DotProductItem; -import com.yahoo.prelude.query.EquivItem; -import com.yahoo.prelude.query.ExactStringItem; -import com.yahoo.prelude.query.IntItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.Limit; -import com.yahoo.prelude.query.GeoLocationItem; -import com.yahoo.prelude.query.NearItem; -import com.yahoo.prelude.query.NearestNeighborItem; -import com.yahoo.prelude.query.NotItem; -import com.yahoo.prelude.query.ONearItem; -import com.yahoo.prelude.query.OrItem; -import com.yahoo.prelude.query.PhraseItem; -import com.yahoo.prelude.query.PredicateQueryItem; -import com.yahoo.prelude.query.PrefixItem; -import com.yahoo.prelude.query.RangeItem; -import com.yahoo.prelude.query.RankItem; -import com.yahoo.prelude.query.RegExpItem; -import com.yahoo.prelude.query.SameElementItem; -import com.yahoo.prelude.query.SegmentingRule; -import com.yahoo.prelude.query.Substring; -import com.yahoo.prelude.query.SubstringItem; -import com.yahoo.prelude.query.SuffixItem; -import com.yahoo.prelude.query.TaggableItem; -import com.yahoo.prelude.query.WandItem; -import com.yahoo.prelude.query.WeakAndItem; -import com.yahoo.prelude.query.WeightedSetItem; -import com.yahoo.prelude.query.WordAlternativesItem; -import com.yahoo.prelude.query.WordItem; import com.yahoo.search.grouping.request.GroupingOperation; import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.Parser; @@ -60,65 +29,13 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import static com.yahoo.search.yql.YqlParser.*; import static com.yahoo.slime.Type.ARRAY; import static com.yahoo.slime.Type.DOUBLE; import static com.yahoo.slime.Type.LONG; import static com.yahoo.slime.Type.OBJECT; import static com.yahoo.slime.Type.STRING; -import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; -import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; -import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; -import static com.yahoo.search.yql.YqlParser.ANNOTATIONS; -import static com.yahoo.search.yql.YqlParser.APPROXIMATE; -import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER; -import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; -import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; -import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; -import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS; -import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER; -import static com.yahoo.search.yql.YqlParser.DISTANCE; -import static com.yahoo.search.yql.YqlParser.DISTANCE_THRESHOLD; -import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; -import static com.yahoo.search.yql.YqlParser.EQUIV; -import static com.yahoo.search.yql.YqlParser.FILTER; -import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; -import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; -import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS; -import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; -import static com.yahoo.search.yql.YqlParser.LABEL; -import static com.yahoo.search.yql.YqlParser.NEAR; -import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; -import static com.yahoo.search.yql.YqlParser.NFKC; -import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; -import static com.yahoo.search.yql.YqlParser.ONEAR; -import static com.yahoo.search.yql.YqlParser.ORIGIN; -import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; -import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; -import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; -import static com.yahoo.search.yql.YqlParser.PHRASE; -import static com.yahoo.search.yql.YqlParser.PREDICATE; -import static com.yahoo.search.yql.YqlParser.PREFIX; -import static com.yahoo.search.yql.YqlParser.RANGE; -import static com.yahoo.search.yql.YqlParser.RANK; -import static com.yahoo.search.yql.YqlParser.RANKED; -import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; -import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; -import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; -import static com.yahoo.search.yql.YqlParser.STEM; -import static com.yahoo.search.yql.YqlParser.SUBSTRING; -import static com.yahoo.search.yql.YqlParser.SUFFIX; -import static com.yahoo.search.yql.YqlParser.TARGET_HITS; -import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; -import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; -import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; -import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; -import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE; -import static com.yahoo.search.yql.YqlParser.WAND; -import static com.yahoo.search.yql.YqlParser.WEAK_AND; -import static com.yahoo.search.yql.YqlParser.WEIGHT; -import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; - /** * The Select query language. * @@ -926,6 +843,8 @@ public class SelectParser implements Parser { return instantiateONearItem(field, key, value); case EQUIV: return instantiateEquivItem(field, key, value); + case FUZZY: + return instantiateFuzzyItem(field, key, value); case ALTERNATIVES: return instantiateWordAlternativesItem(field, key, value); default: @@ -1155,6 +1074,15 @@ public class SelectParser implements Parser { return leafStyleSettings(getAnnotations(value), equiv); } + private Item instantiateFuzzyItem(String field, String key, Inspector value) { + HashMap<Integer, Inspector> children = childMap(value); + Preconditions.checkArgument(children.size() == 1, "Expected 1 argument, got %s.", children.size()); + String wordData = children.get(0).asString(); + FuzzyItem fuzzy = new FuzzyItem(field, true, wordData); + + return leafStyleSettings(getAnnotations(value), fuzzy); + } + private Item instantiateWordAlternativesItem(String field, String key, Inspector value) { HashMap<Integer, Inspector> children = childMap(value); Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size()); diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index cc441eb0c3d..afc903ba7cf 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -1,55 +1,6 @@ // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; -import static com.yahoo.search.yql.YqlParser.ACCENT_DROP; -import static com.yahoo.search.yql.YqlParser.ALTERNATIVES; -import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING; -import static com.yahoo.search.yql.YqlParser.BOUNDS; -import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN; -import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN; -import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN; -import static com.yahoo.search.yql.YqlParser.CONNECTION_ID; -import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT; -import static com.yahoo.search.yql.YqlParser.CONNECTIVITY; -import static com.yahoo.search.yql.YqlParser.DISTANCE; -import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT; -import static com.yahoo.search.yql.YqlParser.END_ANCHOR; -import static com.yahoo.search.yql.YqlParser.EQUIV; -import static com.yahoo.search.yql.YqlParser.FILTER; -import static com.yahoo.search.yql.YqlParser.GEO_LOCATION; -import static com.yahoo.search.yql.YqlParser.HIT_LIMIT; -import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS; -import static com.yahoo.search.yql.YqlParser.LABEL; -import static com.yahoo.search.yql.YqlParser.NEAR; -import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR; -import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE; -import static com.yahoo.search.yql.YqlParser.ONEAR; -import static com.yahoo.search.yql.YqlParser.ORIGIN; -import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH; -import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET; -import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL; -import static com.yahoo.search.yql.YqlParser.PHRASE; -import static com.yahoo.search.yql.YqlParser.PREFIX; -import static com.yahoo.search.yql.YqlParser.RANGE; -import static com.yahoo.search.yql.YqlParser.RANK; -import static com.yahoo.search.yql.YqlParser.RANKED; -import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT; -import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD; -import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE; -import static com.yahoo.search.yql.YqlParser.START_ANCHOR; -import static com.yahoo.search.yql.YqlParser.STEM; -import static com.yahoo.search.yql.YqlParser.SUBSTRING; -import static com.yahoo.search.yql.YqlParser.SUFFIX; -import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS; -import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR; -import static com.yahoo.search.yql.YqlParser.UNIQUE_ID; -import static com.yahoo.search.yql.YqlParser.URI; -import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA; -import static com.yahoo.search.yql.YqlParser.WAND; -import static com.yahoo.search.yql.YqlParser.WEAK_AND; -import static com.yahoo.search.yql.YqlParser.WEIGHT; -import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET; - import java.util.ArrayDeque; import java.util.ArrayList; import java.util.Collection; @@ -64,51 +15,15 @@ import java.util.Map; import java.util.Map.Entry; import com.google.common.collect.ImmutableMap; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.AndSegmentItem; -import com.yahoo.prelude.query.BoolItem; -import com.yahoo.prelude.query.DotProductItem; -import com.yahoo.prelude.query.EquivItem; -import com.yahoo.prelude.query.FalseItem; -import com.yahoo.prelude.query.ExactStringItem; -import com.yahoo.prelude.query.IndexedItem; -import com.yahoo.prelude.query.IntItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.GeoLocationItem; -import com.yahoo.prelude.query.MarkerWordItem; -import com.yahoo.prelude.query.NearItem; -import com.yahoo.prelude.query.NearestNeighborItem; -import com.yahoo.prelude.query.NotItem; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.ONearItem; -import com.yahoo.prelude.query.OrItem; -import com.yahoo.prelude.query.PhraseItem; -import com.yahoo.prelude.query.PhraseSegmentItem; -import com.yahoo.prelude.query.PredicateQueryItem; -import com.yahoo.prelude.query.PrefixItem; -import com.yahoo.prelude.query.RangeItem; -import com.yahoo.prelude.query.RankItem; -import com.yahoo.prelude.query.RegExpItem; -import com.yahoo.prelude.query.SameElementItem; -import com.yahoo.prelude.query.SegmentingRule; -import com.yahoo.prelude.query.Substring; -import com.yahoo.prelude.query.SubstringItem; -import com.yahoo.prelude.query.SuffixItem; -import com.yahoo.prelude.query.TaggableItem; -import com.yahoo.prelude.query.ToolBox; +import com.yahoo.prelude.query.*; import com.yahoo.prelude.query.ToolBox.QueryVisitor; -import com.yahoo.prelude.query.TrueItem; -import com.yahoo.prelude.query.UriItem; -import com.yahoo.prelude.query.WandItem; -import com.yahoo.prelude.query.WeakAndItem; -import com.yahoo.prelude.query.WeightedSetItem; -import com.yahoo.prelude.query.WordAlternativesItem; -import com.yahoo.prelude.query.WordItem; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; import com.yahoo.search.grouping.GroupingRequest; import com.yahoo.search.query.QueryTree; +import static com.yahoo.search.yql.YqlParser.*; + /** * Serialize Vespa query trees to YQL+ strings. * @@ -517,6 +432,32 @@ public class VespaSerializer { } } + private static class FuzzySerializer extends Serializer<FuzzyItem> { + + @Override + void onExit(StringBuilder destination, FuzzyItem item) { } + + @Override + boolean serialize(StringBuilder destination, FuzzyItem fuzzy) { + String annotations = leafAnnotations(fuzzy); + destination.append(normalizeIndexName(fuzzy.getIndexName())).append(" contains "); + + if (annotations.length() > 0) { + destination.append('(').append(annotations); + } + + destination.append(FUZZY).append('('); + destination.append('"'); + escape(fuzzy.getIndexedString(), destination).append('"'); + destination.append(')'); + + if (annotations.length() > 0) { + destination.append(')'); + } + return false; + } + } + private static class ONearSerializer extends Serializer<ONearItem> { @Override @@ -1239,6 +1180,7 @@ public class VespaSerializer { dispatchBuilder.put(WordItem.class, new WordSerializer()); dispatchBuilder.put(RegExpItem.class, new RegExpSerializer()); dispatchBuilder.put(UriItem.class, new UriSerializer()); + dispatchBuilder.put(FuzzyItem.class, new FuzzySerializer()); dispatch = ImmutableMap.copyOf(dispatchBuilder); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index 26508fec3c4..99a449026d8 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -27,48 +27,8 @@ import com.yahoo.language.process.Normalizer; import com.yahoo.language.process.Segmenter; import com.yahoo.prelude.IndexFacts; import com.yahoo.prelude.Location; -import com.yahoo.prelude.query.AndItem; -import com.yahoo.prelude.query.AndSegmentItem; -import com.yahoo.prelude.query.BoolItem; -import com.yahoo.prelude.query.CompositeItem; -import com.yahoo.prelude.query.DotProductItem; -import com.yahoo.prelude.query.EquivItem; -import com.yahoo.prelude.query.FalseItem; -import com.yahoo.prelude.query.ExactStringItem; -import com.yahoo.prelude.query.IntItem; -import com.yahoo.prelude.query.Item; -import com.yahoo.prelude.query.Limit; -import com.yahoo.prelude.query.GeoLocationItem; -import com.yahoo.prelude.query.NearItem; -import com.yahoo.prelude.query.NearestNeighborItem; -import com.yahoo.prelude.query.NotItem; -import com.yahoo.prelude.query.NullItem; -import com.yahoo.prelude.query.ONearItem; -import com.yahoo.prelude.query.OrItem; -import com.yahoo.prelude.query.PhraseItem; -import com.yahoo.prelude.query.PhraseSegmentItem; -import com.yahoo.prelude.query.PredicateQueryItem; -import com.yahoo.prelude.query.PrefixItem; -import com.yahoo.prelude.query.RangeItem; -import com.yahoo.prelude.query.RankItem; -import com.yahoo.prelude.query.RegExpItem; -import com.yahoo.prelude.query.SameElementItem; -import com.yahoo.prelude.query.SegmentItem; -import com.yahoo.prelude.query.SegmentingRule; -import com.yahoo.prelude.query.Substring; -import com.yahoo.prelude.query.SubstringItem; -import com.yahoo.prelude.query.SuffixItem; -import com.yahoo.prelude.query.TaggableItem; -import com.yahoo.prelude.query.TermItem; -import com.yahoo.prelude.query.ToolBox; +import com.yahoo.prelude.query.*; import com.yahoo.prelude.query.ToolBox.QueryVisitor; -import com.yahoo.prelude.query.TrueItem; -import com.yahoo.prelude.query.UriItem; -import com.yahoo.prelude.query.WandItem; -import com.yahoo.prelude.query.WeakAndItem; -import com.yahoo.prelude.query.WeightedSetItem; -import com.yahoo.prelude.query.WordAlternativesItem; -import com.yahoo.prelude.query.WordItem; import com.yahoo.processing.IllegalInputException; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; @@ -192,6 +152,7 @@ public class YqlParser implements Parser { public static final String WEAK_AND = "weakAnd"; public static final String WEIGHT = "weight"; public static final String WEIGHTED_SET = "weightedSet"; + public static final String FUZZY = "fuzzy"; private final IndexFacts indexFacts; private final List<ConnectedItem> connectedItems = new ArrayList<>(); @@ -1171,7 +1132,7 @@ public class YqlParser implements Parser { assertHasOperator(ast, ExpressionOperator.CONTAINS); String field = getIndex(ast.getArgument(0)); if (userQuery != null && indexFactsSession.getIndex(field).isAttribute()) { - userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly", 2); + userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly (unless fuzzy query is used)", 2); } return instantiateLeafItem(field, ast.<OperatorNode<ExpressionOperator>> getArgument(1)); } @@ -1298,11 +1259,23 @@ public class YqlParser implements Parser { return instantiateWordAlternativesItem(field, ast); case URI: return instantiateUriItem(field, ast); + case FUZZY: + return instantiateFuzzyItem(field, ast); default: - throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT, URI); + throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT, URI, FUZZY); } } + private Item instantiateFuzzyItem(String field, OperatorNode<ExpressionOperator> ast) { + List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); + Preconditions.checkArgument(args.size() == 1, "Expected 1 argument, got %s.", args.size()); + + String wordData = getStringContents(args.get(0)); + + FuzzyItem fuzzy = new FuzzyItem(field, true, wordData); + return leafStyleSettings(ast, fuzzy); + } + private Item instantiateEquivItem(String field, OperatorNode<ExpressionOperator> ast) { List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1); Preconditions.checkArgument(args.size() >= 2, "Expected 2 or more arguments, got %s.", args.size()); |