aboutsummaryrefslogtreecommitdiffstats
path: root/container-search
diff options
context:
space:
mode:
authorAlexey Chernyshev <aleksei@spotify.com>2022-03-10 16:33:07 +0100
committerAlexey Chernyshev <aleksei@spotify.com>2022-03-23 16:20:59 +0100
commitd9805209e3b0e33be3c0cc454c4604043663c1c4 (patch)
tree7446c79f68acd8775233ace4d5a70058f90c8406 /container-search
parenta2b1e6654cabc90ddf7422e58adf641876e5201c (diff)
Introducing fuzzy search
Diffstat (limited to 'container-search')
-rw-r--r--container-search/abi-spec.json25
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java106
-rw-r--r--container-search/src/main/java/com/yahoo/prelude/query/Item.java3
-rw-r--r--container-search/src/main/java/com/yahoo/search/query/SelectParser.java98
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java118
-rw-r--r--container-search/src/main/java/com/yahoo/search/yql/YqlParser.java59
-rw-r--r--container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java6
-rw-r--r--container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java25
-rw-r--r--container-search/src/test/java/com/yahoo/select/SelectTestCase.java6
9 files changed, 211 insertions, 235 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index 73d4b99b382..d634fe7795f 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -541,6 +541,30 @@
],
"fields": []
},
+ "com.yahoo.prelude.query.FuzzyItem": {
+ "superClass": "com.yahoo.prelude.query.TermItem",
+ "interfaces": [],
+ "attributes": [
+ "public"
+ ],
+ "methods": [
+ "public void <init>(java.lang.String, boolean, java.lang.String)",
+ "public void setValue(java.lang.String)",
+ "public java.lang.String getRawWord()",
+ "public boolean isWords()",
+ "public com.yahoo.prelude.query.Item$ItemType getItemType()",
+ "public java.lang.String getName()",
+ "public java.lang.String stringValue()",
+ "public boolean isStemmed()",
+ "public java.lang.String getIndexedString()",
+ "public int getNumWords()",
+ "public boolean equals(java.lang.Object)",
+ "public int hashCode()",
+ "public java.lang.String toString()",
+ "protected void encodeThis(java.nio.ByteBuffer)"
+ ],
+ "fields": []
+ },
"com.yahoo.prelude.query.GeoLocationItem": {
"superClass": "com.yahoo.prelude.query.TermItem",
"interfaces": [],
@@ -742,6 +766,7 @@
"public static final enum com.yahoo.prelude.query.Item$ItemType GEO_LOCATION_TERM",
"public static final enum com.yahoo.prelude.query.Item$ItemType TRUE",
"public static final enum com.yahoo.prelude.query.Item$ItemType FALSE",
+ "public static final enum com.yahoo.prelude.query.Item$ItemType FUZZY",
"public final int code"
]
},
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java
new file mode 100644
index 00000000000..9a710deca67
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/query/FuzzyItem.java
@@ -0,0 +1,106 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.query;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Fuzzy search term
+ *
+ * @author alexeyche
+ */
+public class FuzzyItem extends TermItem {
+ private String fuzzyQuery;
+
+ public FuzzyItem(String indexName, boolean isFromQuery, String fuzzyQuery) {
+ super(indexName, isFromQuery, null);
+ setValue(fuzzyQuery);
+ }
+
+ @Override
+ public void setValue(String value) {
+ this.fuzzyQuery = value;
+ }
+
+ @Override
+ public String getRawWord() {
+ return stringValue();
+ }
+
+ @Override
+ public boolean isWords() {
+ return false;
+ }
+
+ @Override
+ public ItemType getItemType() {
+ return ItemType.FUZZY;
+ }
+
+ @Override
+ public String getName() {
+ return "FUZZY";
+ }
+
+ @Override
+ public String stringValue() {
+ return fuzzyQuery;
+ }
+
+ @Override
+ public boolean isStemmed() {
+ return false;
+ }
+
+ @Override
+ public String getIndexedString() {
+ return stringValue();
+ }
+
+ @Override
+ public int getNumWords() {
+ return 1;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!super.equals(obj)) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ FuzzyItem other = (FuzzyItem) obj;
+ if (fuzzyQuery == null) {
+ if (other.fuzzyQuery != null) {
+ return false;
+ }
+ } else if (!fuzzyQuery.equals(other.fuzzyQuery)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((fuzzyQuery == null) ? 0 : fuzzyQuery.hashCode());
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("FuzzyItem [fuzzyQuery=").append(fuzzyQuery).append("]");
+ return builder.toString();
+ }
+
+ protected void encodeThis(ByteBuffer buffer) {
+ super.encodeThis(buffer);
+ putString(getIndexedString(), buffer);
+ }
+}
+
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java
index 2e0c3cf8593..02b208b6ce1 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java
@@ -56,7 +56,8 @@ public abstract class Item implements Cloneable {
NEAREST_NEIGHBOR(26),
GEO_LOCATION_TERM(27),
TRUE(28),
- FALSE(29);
+ FALSE(29),
+ FUZZY(30);
public final int code;
diff --git a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
index 1805a11ff5e..6f7aff798a5 100644
--- a/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
+++ b/container-search/src/main/java/com/yahoo/search/query/SelectParser.java
@@ -2,6 +2,7 @@
package com.yahoo.search.query;
import com.google.common.base.Preconditions;
+import com.yahoo.prelude.query.*;
import com.yahoo.processing.IllegalInputException;
import com.yahoo.collections.LazyMap;
import com.yahoo.geo.DistanceParser;
@@ -10,38 +11,6 @@ import com.yahoo.language.Language;
import com.yahoo.language.process.Normalizer;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.Location;
-import com.yahoo.prelude.query.AndItem;
-import com.yahoo.prelude.query.BoolItem;
-import com.yahoo.prelude.query.CompositeItem;
-import com.yahoo.prelude.query.DotProductItem;
-import com.yahoo.prelude.query.EquivItem;
-import com.yahoo.prelude.query.ExactStringItem;
-import com.yahoo.prelude.query.IntItem;
-import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.Limit;
-import com.yahoo.prelude.query.GeoLocationItem;
-import com.yahoo.prelude.query.NearItem;
-import com.yahoo.prelude.query.NearestNeighborItem;
-import com.yahoo.prelude.query.NotItem;
-import com.yahoo.prelude.query.ONearItem;
-import com.yahoo.prelude.query.OrItem;
-import com.yahoo.prelude.query.PhraseItem;
-import com.yahoo.prelude.query.PredicateQueryItem;
-import com.yahoo.prelude.query.PrefixItem;
-import com.yahoo.prelude.query.RangeItem;
-import com.yahoo.prelude.query.RankItem;
-import com.yahoo.prelude.query.RegExpItem;
-import com.yahoo.prelude.query.SameElementItem;
-import com.yahoo.prelude.query.SegmentingRule;
-import com.yahoo.prelude.query.Substring;
-import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SuffixItem;
-import com.yahoo.prelude.query.TaggableItem;
-import com.yahoo.prelude.query.WandItem;
-import com.yahoo.prelude.query.WeakAndItem;
-import com.yahoo.prelude.query.WeightedSetItem;
-import com.yahoo.prelude.query.WordAlternativesItem;
-import com.yahoo.prelude.query.WordItem;
import com.yahoo.search.grouping.request.GroupingOperation;
import com.yahoo.search.query.parser.Parsable;
import com.yahoo.search.query.parser.Parser;
@@ -60,65 +29,13 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
+import static com.yahoo.search.yql.YqlParser.*;
import static com.yahoo.slime.Type.ARRAY;
import static com.yahoo.slime.Type.DOUBLE;
import static com.yahoo.slime.Type.LONG;
import static com.yahoo.slime.Type.OBJECT;
import static com.yahoo.slime.Type.STRING;
-import static com.yahoo.search.yql.YqlParser.ACCENT_DROP;
-import static com.yahoo.search.yql.YqlParser.ALTERNATIVES;
-import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING;
-import static com.yahoo.search.yql.YqlParser.ANNOTATIONS;
-import static com.yahoo.search.yql.YqlParser.APPROXIMATE;
-import static com.yahoo.search.yql.YqlParser.ASCENDING_HITS_ORDER;
-import static com.yahoo.search.yql.YqlParser.CONNECTION_ID;
-import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT;
-import static com.yahoo.search.yql.YqlParser.CONNECTIVITY;
-import static com.yahoo.search.yql.YqlParser.DEFAULT_TARGET_NUM_HITS;
-import static com.yahoo.search.yql.YqlParser.DESCENDING_HITS_ORDER;
-import static com.yahoo.search.yql.YqlParser.DISTANCE;
-import static com.yahoo.search.yql.YqlParser.DISTANCE_THRESHOLD;
-import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT;
-import static com.yahoo.search.yql.YqlParser.EQUIV;
-import static com.yahoo.search.yql.YqlParser.FILTER;
-import static com.yahoo.search.yql.YqlParser.GEO_LOCATION;
-import static com.yahoo.search.yql.YqlParser.HIT_LIMIT;
-import static com.yahoo.search.yql.YqlParser.HNSW_EXPLORE_ADDITIONAL_HITS;
-import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS;
-import static com.yahoo.search.yql.YqlParser.LABEL;
-import static com.yahoo.search.yql.YqlParser.NEAR;
-import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR;
-import static com.yahoo.search.yql.YqlParser.NFKC;
-import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE;
-import static com.yahoo.search.yql.YqlParser.ONEAR;
-import static com.yahoo.search.yql.YqlParser.ORIGIN;
-import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH;
-import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET;
-import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL;
-import static com.yahoo.search.yql.YqlParser.PHRASE;
-import static com.yahoo.search.yql.YqlParser.PREDICATE;
-import static com.yahoo.search.yql.YqlParser.PREFIX;
-import static com.yahoo.search.yql.YqlParser.RANGE;
-import static com.yahoo.search.yql.YqlParser.RANK;
-import static com.yahoo.search.yql.YqlParser.RANKED;
-import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT;
-import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD;
-import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE;
-import static com.yahoo.search.yql.YqlParser.STEM;
-import static com.yahoo.search.yql.YqlParser.SUBSTRING;
-import static com.yahoo.search.yql.YqlParser.SUFFIX;
-import static com.yahoo.search.yql.YqlParser.TARGET_HITS;
-import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS;
-import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR;
-import static com.yahoo.search.yql.YqlParser.UNIQUE_ID;
-import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA;
-import static com.yahoo.search.yql.YqlParser.USER_INPUT_LANGUAGE;
-import static com.yahoo.search.yql.YqlParser.WAND;
-import static com.yahoo.search.yql.YqlParser.WEAK_AND;
-import static com.yahoo.search.yql.YqlParser.WEIGHT;
-import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET;
-
/**
* The Select query language.
*
@@ -926,6 +843,8 @@ public class SelectParser implements Parser {
return instantiateONearItem(field, key, value);
case EQUIV:
return instantiateEquivItem(field, key, value);
+ case FUZZY:
+ return instantiateFuzzyItem(field, key, value);
case ALTERNATIVES:
return instantiateWordAlternativesItem(field, key, value);
default:
@@ -1155,6 +1074,15 @@ public class SelectParser implements Parser {
return leafStyleSettings(getAnnotations(value), equiv);
}
+ private Item instantiateFuzzyItem(String field, String key, Inspector value) {
+ HashMap<Integer, Inspector> children = childMap(value);
+ Preconditions.checkArgument(children.size() == 1, "Expected 1 argument, got %s.", children.size());
+ String wordData = children.get(0).asString();
+ FuzzyItem fuzzy = new FuzzyItem(field, true, wordData);
+
+ return leafStyleSettings(getAnnotations(value), fuzzy);
+ }
+
private Item instantiateWordAlternativesItem(String field, String key, Inspector value) {
HashMap<Integer, Inspector> children = childMap(value);
Preconditions.checkArgument(children.size() >= 1, "Expected 1 or more arguments, got %s.", children.size());
diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
index cc441eb0c3d..afc903ba7cf 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java
@@ -1,55 +1,6 @@
// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
package com.yahoo.search.yql;
-import static com.yahoo.search.yql.YqlParser.ACCENT_DROP;
-import static com.yahoo.search.yql.YqlParser.ALTERNATIVES;
-import static com.yahoo.search.yql.YqlParser.AND_SEGMENTING;
-import static com.yahoo.search.yql.YqlParser.BOUNDS;
-import static com.yahoo.search.yql.YqlParser.BOUNDS_LEFT_OPEN;
-import static com.yahoo.search.yql.YqlParser.BOUNDS_OPEN;
-import static com.yahoo.search.yql.YqlParser.BOUNDS_RIGHT_OPEN;
-import static com.yahoo.search.yql.YqlParser.CONNECTION_ID;
-import static com.yahoo.search.yql.YqlParser.CONNECTION_WEIGHT;
-import static com.yahoo.search.yql.YqlParser.CONNECTIVITY;
-import static com.yahoo.search.yql.YqlParser.DISTANCE;
-import static com.yahoo.search.yql.YqlParser.DOT_PRODUCT;
-import static com.yahoo.search.yql.YqlParser.END_ANCHOR;
-import static com.yahoo.search.yql.YqlParser.EQUIV;
-import static com.yahoo.search.yql.YqlParser.FILTER;
-import static com.yahoo.search.yql.YqlParser.GEO_LOCATION;
-import static com.yahoo.search.yql.YqlParser.HIT_LIMIT;
-import static com.yahoo.search.yql.YqlParser.IMPLICIT_TRANSFORMS;
-import static com.yahoo.search.yql.YqlParser.LABEL;
-import static com.yahoo.search.yql.YqlParser.NEAR;
-import static com.yahoo.search.yql.YqlParser.NEAREST_NEIGHBOR;
-import static com.yahoo.search.yql.YqlParser.NORMALIZE_CASE;
-import static com.yahoo.search.yql.YqlParser.ONEAR;
-import static com.yahoo.search.yql.YqlParser.ORIGIN;
-import static com.yahoo.search.yql.YqlParser.ORIGIN_LENGTH;
-import static com.yahoo.search.yql.YqlParser.ORIGIN_OFFSET;
-import static com.yahoo.search.yql.YqlParser.ORIGIN_ORIGINAL;
-import static com.yahoo.search.yql.YqlParser.PHRASE;
-import static com.yahoo.search.yql.YqlParser.PREFIX;
-import static com.yahoo.search.yql.YqlParser.RANGE;
-import static com.yahoo.search.yql.YqlParser.RANK;
-import static com.yahoo.search.yql.YqlParser.RANKED;
-import static com.yahoo.search.yql.YqlParser.SAME_ELEMENT;
-import static com.yahoo.search.yql.YqlParser.SCORE_THRESHOLD;
-import static com.yahoo.search.yql.YqlParser.SIGNIFICANCE;
-import static com.yahoo.search.yql.YqlParser.START_ANCHOR;
-import static com.yahoo.search.yql.YqlParser.STEM;
-import static com.yahoo.search.yql.YqlParser.SUBSTRING;
-import static com.yahoo.search.yql.YqlParser.SUFFIX;
-import static com.yahoo.search.yql.YqlParser.TARGET_NUM_HITS;
-import static com.yahoo.search.yql.YqlParser.THRESHOLD_BOOST_FACTOR;
-import static com.yahoo.search.yql.YqlParser.UNIQUE_ID;
-import static com.yahoo.search.yql.YqlParser.URI;
-import static com.yahoo.search.yql.YqlParser.USE_POSITION_DATA;
-import static com.yahoo.search.yql.YqlParser.WAND;
-import static com.yahoo.search.yql.YqlParser.WEAK_AND;
-import static com.yahoo.search.yql.YqlParser.WEIGHT;
-import static com.yahoo.search.yql.YqlParser.WEIGHTED_SET;
-
import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Collection;
@@ -64,51 +15,15 @@ import java.util.Map;
import java.util.Map.Entry;
import com.google.common.collect.ImmutableMap;
-import com.yahoo.prelude.query.AndItem;
-import com.yahoo.prelude.query.AndSegmentItem;
-import com.yahoo.prelude.query.BoolItem;
-import com.yahoo.prelude.query.DotProductItem;
-import com.yahoo.prelude.query.EquivItem;
-import com.yahoo.prelude.query.FalseItem;
-import com.yahoo.prelude.query.ExactStringItem;
-import com.yahoo.prelude.query.IndexedItem;
-import com.yahoo.prelude.query.IntItem;
-import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.GeoLocationItem;
-import com.yahoo.prelude.query.MarkerWordItem;
-import com.yahoo.prelude.query.NearItem;
-import com.yahoo.prelude.query.NearestNeighborItem;
-import com.yahoo.prelude.query.NotItem;
-import com.yahoo.prelude.query.NullItem;
-import com.yahoo.prelude.query.ONearItem;
-import com.yahoo.prelude.query.OrItem;
-import com.yahoo.prelude.query.PhraseItem;
-import com.yahoo.prelude.query.PhraseSegmentItem;
-import com.yahoo.prelude.query.PredicateQueryItem;
-import com.yahoo.prelude.query.PrefixItem;
-import com.yahoo.prelude.query.RangeItem;
-import com.yahoo.prelude.query.RankItem;
-import com.yahoo.prelude.query.RegExpItem;
-import com.yahoo.prelude.query.SameElementItem;
-import com.yahoo.prelude.query.SegmentingRule;
-import com.yahoo.prelude.query.Substring;
-import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SuffixItem;
-import com.yahoo.prelude.query.TaggableItem;
-import com.yahoo.prelude.query.ToolBox;
+import com.yahoo.prelude.query.*;
import com.yahoo.prelude.query.ToolBox.QueryVisitor;
-import com.yahoo.prelude.query.TrueItem;
-import com.yahoo.prelude.query.UriItem;
-import com.yahoo.prelude.query.WandItem;
-import com.yahoo.prelude.query.WeakAndItem;
-import com.yahoo.prelude.query.WeightedSetItem;
-import com.yahoo.prelude.query.WordAlternativesItem;
-import com.yahoo.prelude.query.WordItem;
import com.yahoo.search.Query;
import com.yahoo.search.grouping.Continuation;
import com.yahoo.search.grouping.GroupingRequest;
import com.yahoo.search.query.QueryTree;
+import static com.yahoo.search.yql.YqlParser.*;
+
/**
* Serialize Vespa query trees to YQL+ strings.
*
@@ -517,6 +432,32 @@ public class VespaSerializer {
}
}
+ private static class FuzzySerializer extends Serializer<FuzzyItem> {
+
+ @Override
+ void onExit(StringBuilder destination, FuzzyItem item) { }
+
+ @Override
+ boolean serialize(StringBuilder destination, FuzzyItem fuzzy) {
+ String annotations = leafAnnotations(fuzzy);
+ destination.append(normalizeIndexName(fuzzy.getIndexName())).append(" contains ");
+
+ if (annotations.length() > 0) {
+ destination.append('(').append(annotations);
+ }
+
+ destination.append(FUZZY).append('(');
+ destination.append('"');
+ escape(fuzzy.getIndexedString(), destination).append('"');
+ destination.append(')');
+
+ if (annotations.length() > 0) {
+ destination.append(')');
+ }
+ return false;
+ }
+ }
+
private static class ONearSerializer extends Serializer<ONearItem> {
@Override
@@ -1239,6 +1180,7 @@ public class VespaSerializer {
dispatchBuilder.put(WordItem.class, new WordSerializer());
dispatchBuilder.put(RegExpItem.class, new RegExpSerializer());
dispatchBuilder.put(UriItem.class, new UriSerializer());
+ dispatchBuilder.put(FuzzyItem.class, new FuzzySerializer());
dispatch = ImmutableMap.copyOf(dispatchBuilder);
}
diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
index 26508fec3c4..99a449026d8 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java
@@ -27,48 +27,8 @@ import com.yahoo.language.process.Normalizer;
import com.yahoo.language.process.Segmenter;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.Location;
-import com.yahoo.prelude.query.AndItem;
-import com.yahoo.prelude.query.AndSegmentItem;
-import com.yahoo.prelude.query.BoolItem;
-import com.yahoo.prelude.query.CompositeItem;
-import com.yahoo.prelude.query.DotProductItem;
-import com.yahoo.prelude.query.EquivItem;
-import com.yahoo.prelude.query.FalseItem;
-import com.yahoo.prelude.query.ExactStringItem;
-import com.yahoo.prelude.query.IntItem;
-import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.Limit;
-import com.yahoo.prelude.query.GeoLocationItem;
-import com.yahoo.prelude.query.NearItem;
-import com.yahoo.prelude.query.NearestNeighborItem;
-import com.yahoo.prelude.query.NotItem;
-import com.yahoo.prelude.query.NullItem;
-import com.yahoo.prelude.query.ONearItem;
-import com.yahoo.prelude.query.OrItem;
-import com.yahoo.prelude.query.PhraseItem;
-import com.yahoo.prelude.query.PhraseSegmentItem;
-import com.yahoo.prelude.query.PredicateQueryItem;
-import com.yahoo.prelude.query.PrefixItem;
-import com.yahoo.prelude.query.RangeItem;
-import com.yahoo.prelude.query.RankItem;
-import com.yahoo.prelude.query.RegExpItem;
-import com.yahoo.prelude.query.SameElementItem;
-import com.yahoo.prelude.query.SegmentItem;
-import com.yahoo.prelude.query.SegmentingRule;
-import com.yahoo.prelude.query.Substring;
-import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SuffixItem;
-import com.yahoo.prelude.query.TaggableItem;
-import com.yahoo.prelude.query.TermItem;
-import com.yahoo.prelude.query.ToolBox;
+import com.yahoo.prelude.query.*;
import com.yahoo.prelude.query.ToolBox.QueryVisitor;
-import com.yahoo.prelude.query.TrueItem;
-import com.yahoo.prelude.query.UriItem;
-import com.yahoo.prelude.query.WandItem;
-import com.yahoo.prelude.query.WeakAndItem;
-import com.yahoo.prelude.query.WeightedSetItem;
-import com.yahoo.prelude.query.WordAlternativesItem;
-import com.yahoo.prelude.query.WordItem;
import com.yahoo.processing.IllegalInputException;
import com.yahoo.search.Query;
import com.yahoo.search.grouping.Continuation;
@@ -192,6 +152,7 @@ public class YqlParser implements Parser {
public static final String WEAK_AND = "weakAnd";
public static final String WEIGHT = "weight";
public static final String WEIGHTED_SET = "weightedSet";
+ public static final String FUZZY = "fuzzy";
private final IndexFacts indexFacts;
private final List<ConnectedItem> connectedItems = new ArrayList<>();
@@ -1171,7 +1132,7 @@ public class YqlParser implements Parser {
assertHasOperator(ast, ExpressionOperator.CONTAINS);
String field = getIndex(ast.getArgument(0));
if (userQuery != null && indexFactsSession.getIndex(field).isAttribute()) {
- userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly", 2);
+ userQuery.trace("Field '" + field + "' is an attribute, 'contains' will only match exactly (unless fuzzy query is used)", 2);
}
return instantiateLeafItem(field, ast.<OperatorNode<ExpressionOperator>> getArgument(1));
}
@@ -1298,11 +1259,23 @@ public class YqlParser implements Parser {
return instantiateWordAlternativesItem(field, ast);
case URI:
return instantiateUriItem(field, ast);
+ case FUZZY:
+ return instantiateFuzzyItem(field, ast);
default:
- throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT, URI);
+ throw newUnexpectedArgumentException(names.get(0), EQUIV, NEAR, ONEAR, PHRASE, SAME_ELEMENT, URI, FUZZY);
}
}
+ private Item instantiateFuzzyItem(String field, OperatorNode<ExpressionOperator> ast) {
+ List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
+ Preconditions.checkArgument(args.size() == 1, "Expected 1 argument, got %s.", args.size());
+
+ String wordData = getStringContents(args.get(0));
+
+ FuzzyItem fuzzy = new FuzzyItem(field, true, wordData);
+ return leafStyleSettings(ast, fuzzy);
+ }
+
private Item instantiateEquivItem(String field, OperatorNode<ExpressionOperator> ast) {
List<OperatorNode<ExpressionOperator>> args = ast.getArgument(1);
Preconditions.checkArgument(args.size() >= 2, "Expected 2 or more arguments, got %s.", args.size());
diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
index 7057f996041..1269c2a5aef 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
@@ -443,4 +443,10 @@ public class VespaSerializerTestCase {
+ "alternatives({\"trees\": 1.0, \"tree\": 0.7}))"
+ ")");
}
+
+ @Test
+ public void testFuzzy() {
+ parseAndConfirm("foo contains fuzzy(\"a\")");
+ }
+
}
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index 15713dc1f97..a057d6f7c16 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -8,24 +8,7 @@ import com.yahoo.prelude.Index;
import com.yahoo.prelude.IndexFacts;
import com.yahoo.prelude.IndexModel;
import com.yahoo.prelude.SearchDefinition;
-import com.yahoo.prelude.query.AndItem;
-import com.yahoo.prelude.query.BoolItem;
-import com.yahoo.prelude.query.ExactStringItem;
-import com.yahoo.prelude.query.IndexedItem;
-import com.yahoo.prelude.query.Item;
-import com.yahoo.prelude.query.MarkerWordItem;
-import com.yahoo.prelude.query.PhraseItem;
-import com.yahoo.prelude.query.PhraseSegmentItem;
-import com.yahoo.prelude.query.PrefixItem;
-import com.yahoo.prelude.query.QueryCanonicalizer;
-import com.yahoo.prelude.query.RegExpItem;
-import com.yahoo.prelude.query.SegmentingRule;
-import com.yahoo.prelude.query.Substring;
-import com.yahoo.prelude.query.SubstringItem;
-import com.yahoo.prelude.query.SuffixItem;
-import com.yahoo.prelude.query.WeakAndItem;
-import com.yahoo.prelude.query.WordAlternativesItem;
-import com.yahoo.prelude.query.WordItem;
+import com.yahoo.prelude.query.*;
import com.yahoo.prelude.querytransform.QueryRewrite;
import com.yahoo.processing.IllegalInputException;
import com.yahoo.search.Query;
@@ -382,6 +365,12 @@ public class YqlParserTestCase {
}
@Test
+ public void testFuzzy() {
+ assertParse("select foo from bar where baz contains fuzzy(\"a\")",
+ "FUZZY baz:a");
+ }
+
+ @Test
public void testStemming() {
assertTrue(getRootWord("select foo from bar where baz contains " +
"([ {stem: false} ]\"colors\")").isStemmed());
diff --git a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
index 0dcfb8392ef..1c8433541e1 100644
--- a/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
+++ b/container-search/src/test/java/com/yahoo/select/SelectTestCase.java
@@ -674,6 +674,12 @@ public class SelectTestCase {
checkWordAlternativesContent(alternatives);
}
+ @Test
+ public void testFuzzy() {
+ assertParse("{ \"contains\": [\"description\", { \"fuzzy\": [\"a\"] }] }",
+ "FUZZY description:a");
+ }
+
//------------------------------------------------------------------- grouping tests
@Test