diff options
11 files changed, 594 insertions, 8 deletions
diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json index 31b4dd2c920..f2348c58d4d 100644 --- a/container-search/abi-spec.json +++ b/container-search/abi-spec.json @@ -643,6 +643,23 @@ "public static final java.lang.String HIGHLIGHTTERMS" ] }, + "com.yahoo.prelude.query.InItem" : { + "superClass" : "com.yahoo.prelude.query.Item", + "interfaces" : [ ], + "attributes" : [ + "public", + "abstract" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public void setIndexName(java.lang.String)", + "public java.lang.String getIndexName()", + "public java.lang.String getName()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.prelude.query.IndexedItem" : { "superClass" : "java.lang.Object", "interfaces" : [ @@ -781,6 +798,8 @@ "public static final enum com.yahoo.prelude.query.Item$ItemType TRUE", "public static final enum com.yahoo.prelude.query.Item$ItemType FALSE", "public static final enum com.yahoo.prelude.query.Item$ItemType FUZZY", + "public static final enum com.yahoo.prelude.query.Item$ItemType STRING_IN", + "public static final enum com.yahoo.prelude.query.Item$ItemType NUMERIC_IN", "public final int code" ] }, @@ -1068,6 +1087,26 @@ ], "fields" : [ ] }, + "com.yahoo.prelude.query.NumericInItem" : { + "superClass" : "com.yahoo.prelude.query.InItem", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public int encode(java.nio.ByteBuffer)", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getTermCount()", + "protected void appendBodyString(java.lang.StringBuilder)", + "public void addToken(long)", + "public java.util.Collection getTokens()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.prelude.query.ONearItem" : { "superClass" : "com.yahoo.prelude.query.NearItem", "interfaces" : [ ], @@ -1529,6 +1568,26 @@ ], "fields" : [ ] }, + "com.yahoo.prelude.query.StringInItem" : { + "superClass" : "com.yahoo.prelude.query.InItem", + "interfaces" : [ ], + "attributes" : [ + "public" + ], + "methods" : [ + "public void <init>(java.lang.String)", + "public com.yahoo.prelude.query.Item$ItemType getItemType()", + "public int encode(java.nio.ByteBuffer)", + "protected void encodeThis(java.nio.ByteBuffer)", + "public int getTermCount()", + "protected void appendBodyString(java.lang.StringBuilder)", + "public void addToken(java.lang.String)", + "public java.util.Collection getTokens()", + "public boolean equals(java.lang.Object)", + "public int hashCode()" + ], + "fields" : [ ] + }, "com.yahoo.prelude.query.Substring" : { "superClass" : "java.lang.Object", "interfaces" : [ ], diff --git a/container-search/src/main/java/com/yahoo/prelude/query/InItem.java b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java new file mode 100644 index 00000000000..badf9de3d26 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/InItem.java @@ -0,0 +1,43 @@ +package com.yahoo.prelude.query; + +import java.util.Objects; + +import static java.util.Objects.requireNonNullElse; + +/* + * Abstract class representing an IN operator. + */ +public abstract class InItem extends Item { + private String indexName; + public InItem(String indexName) { + this.indexName = requireNonNullElse(indexName, ""); + } + + @Override + public void setIndexName(String index) { + this.indexName = requireNonNullElse(index, ""); + } + public String getIndexName() { + return indexName; + } + + @Override + public String getName() { + return getItemType().name(); + } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! super.equals(o)) return false; + var other = (InItem)o; + if ( ! Objects.equals(this.indexName, other.indexName)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), indexName); + } + +}; diff --git a/container-search/src/main/java/com/yahoo/prelude/query/Item.java b/container-search/src/main/java/com/yahoo/prelude/query/Item.java index e38579be2df..578adaaeaf9 100644 --- a/container-search/src/main/java/com/yahoo/prelude/query/Item.java +++ b/container-search/src/main/java/com/yahoo/prelude/query/Item.java @@ -56,7 +56,9 @@ public abstract class Item implements Cloneable { GEO_LOCATION_TERM(27), TRUE(28), FALSE(29), - FUZZY(30); + FUZZY(30), + STRING_IN(31), + NUMERIC_IN(32); public final int code; @@ -241,8 +243,9 @@ public abstract class Item implements Cloneable { byte FEAT_UNIQUEID = 0b01000000; byte FEAT_FLAGS = -0b10000000; - byte type = (byte) (getCode() & CODE_MASK); - if (type != getCode()) + int code = getCode(); + byte type = code >= CODE_MASK ? CODE_MASK : (byte) code; + if (code >= 0x80 + CODE_MASK) throw new IllegalStateException("must increase number of bytes in serialization format for queries"); if (weight != DEFAULT_WEIGHT) { @@ -257,6 +260,9 @@ public abstract class Item implements Cloneable { } buffer.put(type); + if (code >= CODE_MASK) { + buffer.put((byte) (code - CODE_MASK)); + } if ((type & FEAT_WEIGHT) != 0) { IntegerCompressor.putCompressedNumber(weight, buffer); } diff --git a/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java new file mode 100644 index 00000000000..65d0e7aad18 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/NumericInItem.java @@ -0,0 +1,86 @@ +package com.yahoo.prelude.query; + +import com.yahoo.compress.IntegerCompressor; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/* + * Class representing an IN operator with a set of 64-bit + * integer values. + */ +public class NumericInItem extends InItem { + private Set<Long> tokens; + + public NumericInItem(String indexName) { + super(indexName); + tokens = new HashSet<>(1000); + } + + @Override + public Item.ItemType getItemType() { + return Item.ItemType.NUMERIC_IN; + } + + @Override + public int encode(ByteBuffer buffer) { + encodeThis(buffer); + return 1; + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + IntegerCompressor.putCompressedPositiveNumber(tokens.size(), buffer); + putString(getIndexName(), buffer); + for (var token : tokens) { + buffer.putLong(token); + } + } + + @Override + public int getTermCount() { + return 1; + } + + @Override + protected void appendBodyString(StringBuilder buffer) { + buffer.append(getIndexName()); + buffer.append("{"); + for (var token : tokens) { + buffer.append(token.toString()); + if (token < Integer.MIN_VALUE || token > Integer.MAX_VALUE) { + buffer.append("L"); + } + buffer.append(","); + } + if (!tokens.isEmpty()) { + buffer.deleteCharAt(buffer.length() - 1); // remove extra "," + } + buffer.append("}"); + } + + public void addToken(long token) { + tokens.add(token); + } + + public Collection<Long> getTokens() { return Set.copyOf(tokens); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! super.equals(o)) return false; + var other = (NumericInItem)o; + if ( ! Objects.equals(this.tokens, other.tokens)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tokens); + } + +} diff --git a/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java new file mode 100644 index 00000000000..e27bb10f067 --- /dev/null +++ b/container-search/src/main/java/com/yahoo/prelude/query/StringInItem.java @@ -0,0 +1,84 @@ +package com.yahoo.prelude.query; + +import com.yahoo.compress.IntegerCompressor; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.HashSet; +import java.util.Objects; +import java.util.Set; + +/* + * Class representing an IN operator with a set of string values. + */ +public class StringInItem extends InItem { + private Set<String> tokens; + + public StringInItem(String indexName) { + super(indexName); + tokens = new HashSet<>(1000); + } + + @Override + public ItemType getItemType() { + return ItemType.STRING_IN; + } + + @Override + public int encode(ByteBuffer buffer) { + encodeThis(buffer); + return 1; + } + + @Override + protected void encodeThis(ByteBuffer buffer) { + super.encodeThis(buffer); + IntegerCompressor.putCompressedPositiveNumber(tokens.size(), buffer); + putString(getIndexName(), buffer); + for (var entry : tokens) { + putString(entry, buffer); + } + } + + @Override + public int getTermCount() { + return 1; + } + + @Override + protected void appendBodyString(StringBuilder buffer) { + buffer.append(getIndexName()); + buffer.append("{"); + for (var entry : tokens) { + buffer.append("\""); + buffer.append(entry); + buffer.append("\","); + } + if (!tokens.isEmpty()) { + buffer.deleteCharAt(buffer.length() - 1); // remove extra "," + } + buffer.append("}"); + } + + public void addToken(String token) { + if (token == null) throw new IllegalArgumentException("token must be a string"); + tokens.add(token); + } + + public Collection<String> getTokens() { return Set.copyOf(tokens); } + + @Override + public boolean equals(Object o) { + if (o == this) return true; + if ( ! super.equals(o)) return false; + var other = (StringInItem)o; + if ( ! Objects.equals(this.tokens, other.tokens)) return false; + return true; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), tokens); + } + +} diff --git a/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java b/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java index 1993871aa4c..94e0deb122f 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/ParameterListParser.java @@ -1,6 +1,8 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; +import com.yahoo.prelude.query.NumericInItem; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.WeightedSetItem; import java.util.Arrays; @@ -61,6 +63,41 @@ class ParameterListParser { s.pass('}'); } + public static void addStringTokensFromString(String string, StringInItem out) { + if (string == null) { + return; + } + var s = new ParsableString(string); + while (!s.atEnd()) { + String key; + if (s.passOptional('\'')) { + key = s.stringTo(s.position('\'')); + s.pass('\''); + } + else if (s.passOptional('"')) { + key = s.stringTo(s.position('"')); + s.pass('"'); + } + else { + key = s.stringTo(s.positionOrEnd(',')).trim(); + } + out.addToken(key); + s.passOptional(','); + } + } + + public static void addNumericTokensFromString(String string, NumericInItem out) { + if (string == null) { + return; + } + var s = new ParsableString(string); + while (!s.atEnd()) { + long key = s.longTo(s.positionOrEnd(',')); + out.addToken(key); + s.passOptional(','); + } + } + private static class ParsableString { int position = 0; @@ -142,6 +179,17 @@ class ParameterListParser { throw new IllegalArgumentException("Expected one of " + Arrays.toString(characters) + " after " + position); } + int positionOrEnd(char ... characters) { + int localPosition = position; + while (localPosition < s.length()) { + char nextChar = s.charAt(localPosition); + for (char character : characters) + if (nextChar == character) return localPosition; + localPosition++; + } + return localPosition; + } + boolean atEnd() { return position >= s.length(); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java index 490fc7aa07f..634163bf0c2 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java +++ b/container-search/src/main/java/com/yahoo/search/yql/VespaSerializer.java @@ -84,6 +84,7 @@ import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; @@ -95,6 +96,7 @@ import com.yahoo.prelude.query.RankItem; import com.yahoo.prelude.query.RegExpItem; import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; @@ -1077,6 +1079,51 @@ public class VespaSerializer { } + private static class StringInSerializer extends Serializer<StringInItem> { + @Override + void onExit(StringBuilder destination, StringInItem item) { + + } + + @Override + boolean serialize(StringBuilder destination, StringInItem item) { + destination.append(item.getIndexName()).append(" in ("); + int initLen = destination.length(); + List<String> tokens = new ArrayList<>(item.getTokens()); + Collections.sort(tokens); + for (var token : tokens) { + comma(destination, initLen); + destination.append('"'); + escape(token, destination); + destination.append("\""); + } + destination.append(")"); + return false; + } + } + + private static class NumericInSerializer extends Serializer<NumericInItem> { + @Override + void onExit(StringBuilder destination, NumericInItem item) { + } + + @Override + boolean serialize(StringBuilder destination, NumericInItem item) { + destination.append(item.getIndexName()).append(" in ("); + int initLen = destination.length(); + List<Long> tokens = new ArrayList<>(item.getTokens()); + Collections.sort(tokens); + for (var token : tokens) { + comma(destination, initLen); + destination.append(token.toString()); + if (token < Integer.MIN_VALUE || token > Integer.MAX_VALUE) + destination.append("L"); + } + destination.append(")"); + return false; + } + } + private static class WordSerializer extends Serializer<WordItem> { @Override @@ -1284,6 +1331,8 @@ public class VespaSerializer { dispatchBuilder.put(RegExpItem.class, new RegExpSerializer()); dispatchBuilder.put(UriItem.class, new UriSerializer()); dispatchBuilder.put(FuzzyItem.class, new FuzzySerializer()); + dispatchBuilder.put(StringInItem.class, new StringInSerializer()); + dispatchBuilder.put(NumericInItem.class, new NumericInSerializer()); dispatch = ImmutableMap.copyOf(dispatchBuilder); } diff --git a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java index eaabdf2d2d4..5e1dfb99479 100644 --- a/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java +++ b/container-search/src/main/java/com/yahoo/search/yql/YqlParser.java @@ -44,6 +44,7 @@ import com.yahoo.prelude.query.NearItem; import com.yahoo.prelude.query.NearestNeighborItem; import com.yahoo.prelude.query.NotItem; import com.yahoo.prelude.query.NullItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.OrItem; import com.yahoo.prelude.query.PhraseItem; @@ -56,6 +57,7 @@ import com.yahoo.prelude.query.RegExpItem; import com.yahoo.prelude.query.SameElementItem; import com.yahoo.prelude.query.SegmentItem; import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; @@ -353,10 +355,12 @@ public class YqlParser implements Parser { case CALL -> buildFunctionCall(ast); case LITERAL -> buildLiteral(ast); case NOT -> buildNot(ast); + case IN -> buildIn(ast); default -> throw newUnexpectedArgumentException(ast.getOperator(), ExpressionOperator.AND, ExpressionOperator.CALL, ExpressionOperator.CONTAINS, ExpressionOperator.EQ, ExpressionOperator.GT, ExpressionOperator.GTEQ, + ExpressionOperator.IN, ExpressionOperator.LT, ExpressionOperator.LTEQ, ExpressionOperator.OR); }; @@ -409,6 +413,18 @@ public class YqlParser implements Parser { return fillWeightedSet(ast, args.get(1), new DotProductItem(getIndex(args.get(0)))); } + private Item buildIn(OperatorNode<ExpressionOperator> ast) { + String field = getIndex(ast.getArgument(0)); + boolean stringField = indexFactsSession.getIndex(field).isString(); + Item item = null; + if (stringField) { + item = fillStringIn(ast, ast.getArgument(1), new StringInItem(field)); + } else { + item = fillNumericIn(ast, ast.getArgument(1), new NumericInItem(field)); + } + return item; + } + private ParsedDegree degreesFromArg(OperatorNode<ExpressionOperator> ast, boolean first) { Object arg = null; switch (ast.getOperator()) { @@ -591,6 +607,52 @@ public class YqlParser implements Parser { return leafStyleSettings(ast, out); } + private StringInItem fillStringIn(OperatorNode<ExpressionOperator> ast, + OperatorNode<ExpressionOperator> arg, + StringInItem out) { + assertHasOperator(arg, ExpressionOperator.ARRAY); + List<OperatorNode<ExpressionOperator>> values = arg.getArgument(0); + for (var value : values) { + switch (value.getOperator()) { + case LITERAL -> { + String tokenValue = value.getArgument(0, String.class); + out.addToken(tokenValue); + } + case VARREF -> { + Preconditions.checkState(userQuery != null, "Query properties are not available"); + String varRef = value.getArgument(0, String.class); + ParameterListParser.addStringTokensFromString(userQuery.properties().getString(varRef), out); + } + default -> throw newUnexpectedArgumentException(value.getOperator(), + ExpressionOperator.LITERAL, ExpressionOperator.VARREF); + } + } + return out; + } + + private NumericInItem fillNumericIn(OperatorNode<ExpressionOperator> ast, + OperatorNode<ExpressionOperator> arg, + NumericInItem out) { + assertHasOperator(arg, ExpressionOperator.ARRAY); + List<OperatorNode<ExpressionOperator>> values = arg.getArgument(0); + for (var value : values) { + switch (value.getOperator()) { + case LITERAL -> { + Long tokenValue = value.getArgument(0, Number.class).longValue(); + out.addToken(tokenValue); + } + case VARREF -> { + Preconditions.checkState(userQuery != null, "Query properties are not available"); + String varRef = value.getArgument(0, String.class); + ParameterListParser.addNumericTokensFromString(userQuery.properties().getString(varRef), out); + } + default -> throw newUnexpectedArgumentException(value.getOperator(), + ExpressionOperator.LITERAL, ExpressionOperator.VARREF); + } + } + return out; + } + private static class PrefixExpander extends IndexNameExpander { private final String prefix; public PrefixExpander(String prefix) { diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java index af7cf2e356f..0c434b2f794 100644 --- a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java +++ b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java @@ -5,16 +5,21 @@ import com.yahoo.prelude.query.AndItem; import com.yahoo.prelude.query.EquivItem; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.ONearItem; import com.yahoo.prelude.query.PureWeightedInteger; import com.yahoo.prelude.query.PureWeightedString; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.WeakAndItem; import com.yahoo.prelude.query.WordItem; import org.junit.jupiter.api.Test; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; /** * Item encoding tests @@ -23,10 +28,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals; */ public class ItemEncodingTestCase { - private void assertType(ByteBuffer buffer, int etype, int features) { - byte type = buffer.get(); - assertEquals(etype, type & 0x1f, "Code"); - assertEquals(features, (type & 0xe0) >> 5, "Features"); + private void assertType(ByteBuffer buffer, int etype, int efeatures) { + byte CODE_MASK = 0b00011111; + byte features_and_type = buffer.get(); + int features = (features_and_type & 0xe0) >> 5; + int type = features_and_type & CODE_MASK; + if (type == CODE_MASK) { + byte type_extension = buffer.get(); + assertTrue(type_extension >= 0); + type += type_extension; + } + assertEquals(etype, type, "Code"); + assertEquals(efeatures, features, "Features"); } private void assertWeight(ByteBuffer buffer, int weight) { @@ -325,6 +338,44 @@ public class ItemEncodingTestCase { ; } + @Test + void testStringInItem() { + var a = new StringInItem("default"); + a.addToken("foo"); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + // 2 bytes type, 1 byte item count, 1 byte string len, 7 bytes string content + // 1 byte string len, 4 bytes string content + assertEquals(15, buffer.remaining(), "Serialization size"); + assertType(buffer, 31, 0); + assertEquals(1, buffer.get()); // 1 item + assertString(buffer, "default"); + assertString(buffer, "foo"); + } + + @Test + void testNumericInItem() { + var a = new NumericInItem("default"); + a.addToken(42); + a.addToken(97000000000L); + ByteBuffer buffer = ByteBuffer.allocate(128); + int count = a.encode(buffer); + buffer.flip(); + // 2 bytes type, 1 byte item count, 1 byte string len, 7 bytes string content + // 16 bytes (2 64-bit integer value) + assertEquals(27, buffer.remaining(), "Serialization size"); + assertType(buffer, 32, 0); + assertEquals(2, buffer.get()); // 2 items + assertString(buffer, "default"); + var array = new ArrayList<Long>(); + array.add(buffer.getLong()); + array.add(buffer.getLong()); + Collections.sort(array); + assertEquals(42, array.get(0)); + assertEquals(97000000000L, array.get(1)); + } + private void assertString(ByteBuffer buffer, String word) { assertEquals(word.length(), buffer.get(), "Word length"); for (int i=0; i<word.length(); i++) { diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java index f3612c3f303..6aac2faa4e9 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java @@ -1,6 +1,10 @@ // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root. package com.yahoo.search.yql; +import com.yahoo.prelude.Index; +import com.yahoo.prelude.IndexFacts; +import com.yahoo.prelude.IndexModel; +import com.yahoo.prelude.SearchDefinition; import com.yahoo.prelude.query.SameElementItem; import com.yahoo.search.Query; import com.yahoo.search.grouping.Continuation; @@ -43,6 +47,15 @@ public class VespaSerializerTestCase { parser = null; } + static private IndexFacts createIndexFactsForInTest() { + SearchDefinition sd = new SearchDefinition("sourceA"); + sd.addIndex(new Index("field")); + Index stringIndex = new Index("string"); + stringIndex.setString(true); + sd.addIndex(stringIndex); + return new IndexFacts(new IndexModel(sd)); + } + @Test void requireThatGroupingRequestsAreSerialized() { Query query = new Query(); @@ -451,4 +464,11 @@ public class VespaSerializerTestCase { parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5}fuzzy(\"a\"))"); } + @Test + void testIn() { + parser = new YqlParser(new ParserEnvironment().setIndexFacts(createIndexFactsForInTest())); + parseAndConfirm("field in (2, 3)"); + parseAndConfirm("field in (9000000000L, 12000000000L)"); + parseAndConfirm("string in (\"a\", \"b\")"); + } } diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java index c1194cfc84b..bd29e2afd53 100644 --- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java +++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java @@ -16,12 +16,14 @@ import com.yahoo.prelude.query.IndexedItem; import com.yahoo.prelude.query.Item; import com.yahoo.prelude.query.MarkerWordItem; import com.yahoo.prelude.query.NearestNeighborItem; +import com.yahoo.prelude.query.NumericInItem; import com.yahoo.prelude.query.PhraseItem; import com.yahoo.prelude.query.PhraseSegmentItem; import com.yahoo.prelude.query.PrefixItem; import com.yahoo.prelude.query.QueryCanonicalizer; import com.yahoo.prelude.query.RegExpItem; import com.yahoo.prelude.query.SegmentingRule; +import com.yahoo.prelude.query.StringInItem; import com.yahoo.prelude.query.Substring; import com.yahoo.prelude.query.SubstringItem; import com.yahoo.prelude.query.SuffixItem; @@ -46,6 +48,8 @@ import com.yahoo.search.query.parser.Parsable; import com.yahoo.search.query.parser.ParserEnvironment; import com.yahoo.search.searchchain.Execution; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -63,7 +67,36 @@ import static org.junit.jupiter.api.Assertions.*; */ public class YqlParserTestCase { - private final YqlParser parser = new YqlParser(new ParserEnvironment()); + private YqlParser parser; + + @BeforeEach + public void setUp() throws Exception { + ParserEnvironment env = new ParserEnvironment(); + parser = new YqlParser(env); + } + + @AfterEach + public void tearDown() throws Exception { + parser = null; + } + + static private IndexFacts createIndexFactsForInTest() { + SearchDefinition sd = new SearchDefinition("default"); + sd.addIndex(new Index("field")); + Index stringIndex = new Index("string"); + stringIndex.setString(true); + sd.addIndex(stringIndex); + return new IndexFacts(new IndexModel(sd)); + } + + private static Query createUserQuery() { + var builder = new Query.Builder(); + var query = builder.build(); + // Following two properties are used by testing of IN operator (cf. testIn) + query.properties().set("foostring", "'this', \"might\", work "); + query.properties().set("foonumeric", "26, 25, -11, 24 "); + return query; + } @Test void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() { @@ -1141,6 +1174,51 @@ public class YqlParserTestCase { parse("select * from sources * where (default contains ({stem: false}\"m\") AND default contains ({origin: {original: \"m\'s\", offset: 0, length: 3}, andSegmenting: true}phrase(\"m\", \"s\"))) timeout 472"); } + @Test + void testIn() { + parser = new YqlParser(new ParserEnvironment().setIndexFacts(createIndexFactsForInTest())); + parser.setUserQuery(createUserQuery()); + var query = parse("select * from sources * where field in (42, 22L, -7, @foonumeric)"); + assertNumericInItem("field", new long[]{-11, -7, 22, 24, 25, 26, 42}, query); + parser.setUserQuery(createUserQuery()); + query = parse("select * from sources * where string in ('a','b', @foostring)"); + assertStringInItem("string", new String[]{"a","b","might","this", "work"}, query); + parser.setUserQuery(createUserQuery()); + query = parse("select * from sources * where field in (29.9, -7.4)"); + assertNumericInItem("field", new long[]{-7, 29}, query); + parser.setUserQuery(null); + assertParseFail("select * from sources * where string in ('a', 25L)", + new ClassCastException("Cannot cast java.lang.Long to java.lang.String")); + assertParseFail("select * from sources * where field in ('a', 25L)", + new ClassCastException("Cannot cast java.lang.String to java.lang.Number")); + assertParseFail("select * from sources * where nofield in ('a', 25L)", + new IllegalArgumentException("Field 'nofield' does not exist.")); + assertParseFail("select * from sources * where field not in (25)", + new IllegalArgumentException("Expected AND, CALL, CONTAINS, EQ, GT, GTEQ, IN, LT, LTEQ or OR, got NOT_IN.")); + } + + private static void assertNumericInItem(String field, long[] values, QueryTree query) { + var exp = buildNumericInItem(field, values); + assertEquals(exp, query.getRoot()); + } + + private static void assertStringInItem(String field, String[] values, QueryTree query) { + var exp = buildStringInItem(field, values); + assertEquals(exp, query.getRoot()); + } + + private static NumericInItem buildNumericInItem(String field, long[] values) { + var item = new NumericInItem(field); + for (var value : values) item.addToken(value); + return item; + } + + private static StringInItem buildStringInItem(String field, String[] values) { + var item = new StringInItem(field); + for (var value : values) item.addToken(value); + return item; + } + private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) { boolean startAnchorIsDefault = false; // Always |