Add InItem to com.yahoo.prelude.query.

3 files changed, 154 insertions, 5 deletions

diff --git a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java
index af7cf2e356f..0c434b2f794 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/test/ItemEncodingTestCase.java
@@ -5,16 +5,21 @@ import com.yahoo.prelude.query.AndItem;
 import com.yahoo.prelude.query.EquivItem;
 import com.yahoo.prelude.query.MarkerWordItem;
 import com.yahoo.prelude.query.NearItem;
+import com.yahoo.prelude.query.NumericInItem;
 import com.yahoo.prelude.query.ONearItem;
 import com.yahoo.prelude.query.PureWeightedInteger;
 import com.yahoo.prelude.query.PureWeightedString;
+import com.yahoo.prelude.query.StringInItem;
 import com.yahoo.prelude.query.WeakAndItem;
 import com.yahoo.prelude.query.WordItem;
 import org.junit.jupiter.api.Test;
 
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
 
 /**
  * Item encoding tests
@@ -23,10 +28,18 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
  */
 public class ItemEncodingTestCase {
 
-    private void assertType(ByteBuffer buffer, int etype, int features) {
-        byte type = buffer.get();
-        assertEquals(etype, type & 0x1f, "Code");
-        assertEquals(features, (type & 0xe0) >> 5, "Features");
+    private void assertType(ByteBuffer buffer, int etype, int efeatures) {
+        byte CODE_MASK = 0b00011111;
+        byte features_and_type = buffer.get();
+        int features = (features_and_type & 0xe0) >> 5;
+        int type = features_and_type & CODE_MASK;
+        if (type == CODE_MASK) {
+            byte type_extension = buffer.get();
+            assertTrue(type_extension >= 0);
+            type += type_extension;
+        }
+        assertEquals(etype, type, "Code");
+        assertEquals(efeatures, features, "Features");
     }
 
     private void assertWeight(ByteBuffer buffer, int weight) {
@@ -325,6 +338,44 @@ public class ItemEncodingTestCase {
         ;
     }
 
+    @Test
+    void testStringInItem() {
+        var a = new StringInItem("default");
+        a.addToken("foo");
+        ByteBuffer buffer = ByteBuffer.allocate(128);
+        int count = a.encode(buffer);
+        buffer.flip();
+        // 2 bytes type, 1 byte item count, 1 byte string len, 7 bytes string content
+        // 1 byte string len, 4 bytes string content
+        assertEquals(15, buffer.remaining(), "Serialization size");
+        assertType(buffer, 31, 0);
+        assertEquals(1, buffer.get()); // 1 item
+        assertString(buffer, "default");
+        assertString(buffer, "foo");
+    }
+
+    @Test
+    void testNumericInItem() {
+        var a = new NumericInItem("default");
+        a.addToken(42);
+        a.addToken(97000000000L);
+        ByteBuffer buffer = ByteBuffer.allocate(128);
+        int count = a.encode(buffer);
+        buffer.flip();
+        // 2 bytes type, 1 byte item count, 1 byte string len, 7 bytes string content
+        // 16 bytes (2 64-bit integer value)
+        assertEquals(27, buffer.remaining(), "Serialization size");
+        assertType(buffer, 32, 0);
+        assertEquals(2, buffer.get()); // 2 items
+        assertString(buffer, "default");
+        var array = new ArrayList<Long>();
+        array.add(buffer.getLong());
+        array.add(buffer.getLong());
+        Collections.sort(array);
+        assertEquals(42, array.get(0));
+        assertEquals(97000000000L, array.get(1));
+    }
+
     private void assertString(ByteBuffer buffer, String word) {
         assertEquals(word.length(), buffer.get(), "Word length");
         for (int i=0; i<word.length(); i++) {
diff --git a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
index f3612c3f303..6aac2faa4e9 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/VespaSerializerTestCase.java
@@ -1,6 +1,10 @@
 // Copyright Vespa.ai. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.search.yql;
 
+import com.yahoo.prelude.Index;
+import com.yahoo.prelude.IndexFacts;
+import com.yahoo.prelude.IndexModel;
+import com.yahoo.prelude.SearchDefinition;
 import com.yahoo.prelude.query.SameElementItem;
 import com.yahoo.search.Query;
 import com.yahoo.search.grouping.Continuation;
@@ -43,6 +47,15 @@ public class VespaSerializerTestCase {
         parser = null;
     }
 
+    static private IndexFacts createIndexFactsForInTest() {
+        SearchDefinition sd = new SearchDefinition("sourceA");
+        sd.addIndex(new Index("field"));
+        Index stringIndex = new Index("string");
+        stringIndex.setString(true);
+        sd.addIndex(stringIndex);
+        return new IndexFacts(new IndexModel(sd));
+    }
+
     @Test
     void requireThatGroupingRequestsAreSerialized() {
         Query query = new Query();
@@ -451,4 +464,11 @@ public class VespaSerializerTestCase {
         parseAndConfirm("foo contains ({maxEditDistance:3,prefixLength:5}fuzzy(\"a\"))");
     }
 
+    @Test
+    void testIn() {
+        parser = new YqlParser(new ParserEnvironment().setIndexFacts(createIndexFactsForInTest()));
+        parseAndConfirm("field in (2, 3)");
+        parseAndConfirm("field in (9000000000L, 12000000000L)");
+        parseAndConfirm("string in (\"a\", \"b\")");
+    }
 }
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index c1194cfc84b..bd29e2afd53 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -16,12 +16,14 @@ import com.yahoo.prelude.query.IndexedItem;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.MarkerWordItem;
 import com.yahoo.prelude.query.NearestNeighborItem;
+import com.yahoo.prelude.query.NumericInItem;
 import com.yahoo.prelude.query.PhraseItem;
 import com.yahoo.prelude.query.PhraseSegmentItem;
 import com.yahoo.prelude.query.PrefixItem;
 import com.yahoo.prelude.query.QueryCanonicalizer;
 import com.yahoo.prelude.query.RegExpItem;
 import com.yahoo.prelude.query.SegmentingRule;
+import com.yahoo.prelude.query.StringInItem;
 import com.yahoo.prelude.query.Substring;
 import com.yahoo.prelude.query.SubstringItem;
 import com.yahoo.prelude.query.SuffixItem;
@@ -46,6 +48,8 @@ import com.yahoo.search.query.parser.Parsable;
 import com.yahoo.search.query.parser.ParserEnvironment;
 
 import com.yahoo.search.searchchain.Execution;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
@@ -63,7 +67,36 @@ import static org.junit.jupiter.api.Assertions.*;
  */
 public class YqlParserTestCase {
 
-    private final YqlParser parser = new YqlParser(new ParserEnvironment());
+    private YqlParser parser;
+
+    @BeforeEach
+    public void setUp() throws Exception {
+        ParserEnvironment env = new ParserEnvironment();
+        parser = new YqlParser(env);
+    }
+
+    @AfterEach
+    public void tearDown() throws Exception {
+        parser = null;
+    }
+
+    static private IndexFacts createIndexFactsForInTest() {
+        SearchDefinition sd = new SearchDefinition("default");
+        sd.addIndex(new Index("field"));
+        Index stringIndex = new Index("string");
+        stringIndex.setString(true);
+        sd.addIndex(stringIndex);
+        return new IndexFacts(new IndexModel(sd));
+    }
+
+    private static Query createUserQuery() {
+        var builder = new Query.Builder();
+        var query = builder.build();
+        // Following two properties are used by testing of IN operator (cf. testIn)
+        query.properties().set("foostring", "'this', \"might\", work ");
+        query.properties().set("foonumeric", "26, 25, -11, 24 ");
+        return query;
+    }
 
     @Test
     void failsGracefullyOnMissingQuoteEscapingAndSubsequentUnicodeCharacter() {
@@ -1141,6 +1174,51 @@ public class YqlParserTestCase {
         parse("select * from sources * where (default contains ({stem: false}\"m\") AND default contains ({origin: {original: \"m\'s\", offset: 0, length: 3}, andSegmenting: true}phrase(\"m\", \"s\"))) timeout 472");
     }
 
+    @Test
+    void testIn() {
+        parser = new YqlParser(new ParserEnvironment().setIndexFacts(createIndexFactsForInTest()));
+        parser.setUserQuery(createUserQuery());
+        var query = parse("select * from sources * where field in (42, 22L, -7, @foonumeric)");
+        assertNumericInItem("field", new long[]{-11, -7, 22, 24, 25, 26, 42}, query);
+        parser.setUserQuery(createUserQuery());
+        query = parse("select * from sources * where string in ('a','b', @foostring)");
+        assertStringInItem("string", new String[]{"a","b","might","this", "work"}, query);
+        parser.setUserQuery(createUserQuery());
+        query = parse("select * from sources * where field in (29.9, -7.4)");
+        assertNumericInItem("field", new long[]{-7, 29}, query);
+        parser.setUserQuery(null);
+        assertParseFail("select * from sources * where string in ('a', 25L)",
+                new ClassCastException("Cannot cast java.lang.Long to java.lang.String"));
+        assertParseFail("select * from sources * where field in ('a', 25L)",
+                new ClassCastException("Cannot cast java.lang.String to java.lang.Number"));
+        assertParseFail("select * from sources * where nofield in ('a', 25L)",
+                new IllegalArgumentException("Field 'nofield' does not exist."));
+        assertParseFail("select * from sources * where field not in (25)",
+                new IllegalArgumentException("Expected AND, CALL, CONTAINS, EQ, GT, GTEQ, IN, LT, LTEQ or OR, got NOT_IN."));
+    }
+
+    private static void assertNumericInItem(String field, long[] values, QueryTree query) {
+        var exp = buildNumericInItem(field, values);
+        assertEquals(exp, query.getRoot());
+    }
+
+    private static void assertStringInItem(String field, String[] values, QueryTree query) {
+        var exp = buildStringInItem(field, values);
+        assertEquals(exp, query.getRoot());
+    }
+
+    private static NumericInItem buildNumericInItem(String field, long[] values) {
+        var item = new NumericInItem(field);
+        for (var value : values) item.addToken(value);
+        return item;
+    }
+
+    private static StringInItem buildStringInItem(String field, String[] values) {
+        var item = new StringInItem(field);
+        for (var value : values) item.addToken(value);
+        return item;
+    }
+
     private void assertUrlQuery(String field, Query query, boolean startAnchor, boolean endAnchor, boolean endAnchorIsDefault) {
         boolean startAnchorIsDefault = false; // Always