Add weakAnd grammar and parser

10 files changed, 121 insertions, 62 deletions

diff --git a/container-search/abi-spec.json b/container-search/abi-spec.json
index 43ce1578e04..6201ec4bfd9 100644
--- a/container-search/abi-spec.json
+++ b/container-search/abi-spec.json
@@ -1896,7 +1896,8 @@
       "public static final enum com.yahoo.search.Query$Type WEB",
       "public static final enum com.yahoo.search.Query$Type PROGRAMMATIC",
       "public static final enum com.yahoo.search.Query$Type YQL",
-      "public static final enum com.yahoo.search.Query$Type SELECT"
+      "public static final enum com.yahoo.search.Query$Type SELECT",
+      "public static final enum com.yahoo.search.Query$Type WEAKAND"
     ]
   },
   "com.yahoo.search.Query": {
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
index 3759dbbcbee..80a2320b039 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/AllParser.java
@@ -2,6 +2,7 @@
 package com.yahoo.prelude.query.parser;
 
 import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
 import com.yahoo.prelude.query.IntItem;
 import com.yahoo.prelude.query.Item;
 import com.yahoo.prelude.query.NotItem;
@@ -10,6 +11,7 @@ import com.yahoo.prelude.query.OrItem;
 import com.yahoo.prelude.query.PhraseItem;
 import com.yahoo.prelude.query.QueryCanonicalizer;
 import com.yahoo.prelude.query.RankItem;
+import com.yahoo.prelude.query.WeakAndItem;
 import com.yahoo.search.query.QueryTree;
 import com.yahoo.search.query.parser.ParserEnvironment;
 
@@ -26,8 +28,16 @@ import static com.yahoo.prelude.query.parser.Token.Kind.SPACE;
  */
 public class AllParser extends SimpleParser {
 
-    public AllParser(ParserEnvironment environment) {
+    private final boolean weakAnd;
+
+    /**
+     * Creates an And parser
+     *
+     * @param weakAnd false to parse into AndItem (by default), true to parse to WeakAnd
+     */
+    public AllParser(ParserEnvironment environment, boolean weakAnd) {
         super(environment);
+        this.weakAnd = weakAnd;
     }
 
     @Override
@@ -42,7 +52,7 @@ public class AllParser extends SimpleParser {
 
     protected Item parseItemsBody() {
         // Algorithm: Collect positive, negative, and and'ed items, then combine.
-        AndItem and = null;
+        CompositeItem and = null;
         NotItem not = null; // Store negatives here as we go
         Item current;
 
@@ -88,13 +98,17 @@ public class AllParser extends SimpleParser {
         return root.getRoot() instanceof NullItem ? null : root.getRoot();
     }
 
-    protected AndItem addAnd(Item item, AndItem and) {
+    protected CompositeItem addAnd(Item item, CompositeItem and) {
         if (and == null)
-            and = new AndItem();
+            and = createAnd();
         and.addItem(item);
         return and;
     }
 
+    private CompositeItem createAnd() {
+        return weakAnd ? new WeakAndItem() : new AndItem();
+    }
+
     protected OrItem addOr(Item item, OrItem or) {
         if (or == null)
             or = new OrItem();
@@ -124,7 +138,7 @@ public class AllParser extends SimpleParser {
                 if (item != null) {
                     isComposited = true;
                     if (item instanceof OrItem) { // Turn into And
-                        AndItem and = new AndItem();
+                        CompositeItem and = createAnd();
 
                         for (Iterator<Item> i = ((OrItem) item).getItemIterator(); i.hasNext();) {
                             and.addItem(i.next());
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java
index a7dbaa94fc0..d7c7dec4798 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/WebParser.java
@@ -1,11 +1,14 @@
 // Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
 package com.yahoo.prelude.query.parser;
 
-import com.yahoo.prelude.query.*;
+import com.yahoo.prelude.query.AndItem;
+import com.yahoo.prelude.query.CompositeItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.NotItem;
+import com.yahoo.prelude.query.OrItem;
+import com.yahoo.prelude.query.WordItem;
 import com.yahoo.search.query.parser.ParserEnvironment;
 
-import java.util.Set;
-
 /**
  * Parser for web search queries. Language:
  *
@@ -21,37 +24,38 @@ import java.util.Set;
 public class WebParser extends AllParser {
 
     public WebParser(ParserEnvironment environment) {
-        super(environment);
+        super(environment, false);
     }
 
-    protected @Override Item parseItemsBody() {
+    @Override
+    protected Item parseItemsBody() {
         // Algorithm: Collect positive, negative, and'ed and or'ed elements, then combine.
-        AndItem and=null;
-        OrItem or=null;
-        NotItem not=null; // Store negatives here as we go
+        CompositeItem and = null;
+        OrItem or = null;
+        NotItem not = null; // Store negatives here as we go
         Item current;
 
         // Find all items
         do {
-            current=negativeItem();
-            if (current!=null) {
-                not=addNot(current,not);
+            current = negativeItem();
+            if (current != null) {
+                not = addNot(current, not);
                 continue;
             }
 
-            current=positiveItem();
-            if (current==null)
+            current = positiveItem();
+            if (current == null)
                 current = indexableItem();
 
-            if (current!=null) {
-                if (and!=null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) {
-                    if (or==null)
-                        or=addOr(and,or);
-                    and=new AndItem();
+            if (current != null) {
+                if (and != null && (current instanceof WordItem) && "OR".equals(((WordItem)current).getRawWord())) {
+                    if (or == null)
+                        or = addOr(and, or);
+                    and = new AndItem();
                     or.addItem(and);
                 }
                 else {
-                    and=addAnd(current,and);
+                    and = addAnd(current, and);
                 }
             }
 
@@ -60,21 +64,17 @@ public class WebParser extends AllParser {
         } while (tokens.hasNext());
 
         // Combine the items
-        Item topLevel=and;
+        Item topLevel = and;
 
-        if (or!=null)
-            topLevel=or;
+        if (or != null)
+            topLevel = or;
 
-        if (not!=null && topLevel!=null) {
+        if (not != null && topLevel != null) {
             not.setPositiveItem(topLevel);
-            topLevel=not;
+            topLevel = not;
         }
 
         return simplifyUnnecessaryComposites(topLevel);
     }
 
-    protected void setSubmodeFromIndex(String indexName, Set<String> searchDefinitions) {
-        // No submodes in this language
-    }
-
 }
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 786a0d0e04f..623c38fa9f0 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -108,7 +108,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         WEB(4,"web"),
         PROGRAMMATIC(5, "prog"),
         YQL(6, "yql"),
-        SELECT(7, "select");
+        SELECT(7, "select"),
+        WEAKAND(8, "weakAnd");
 
         private final int intValue;
         private final String stringValue;
@@ -123,7 +124,7 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
             for (Type type : Type.values())
                 if (type.stringValue.equals(typeString))
                     return type;
-            return ALL;
+            throw new IllegalArgumentException("No query type '" + typeString + "'");
         }
 
         public int asInt() { return intValue; }
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
index a64af7658cc..f9b8f1785db 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
@@ -29,7 +29,7 @@ public final class ParserFactory {
     public static Parser newInstance(Query.Type type, ParserEnvironment environment) {
         switch (type) {
             case ALL:
-                return new AllParser(environment);
+                return new AllParser(environment, false);
             case ANY:
                 return new AnyParser(environment);
             case PHRASE:
@@ -44,6 +44,8 @@ public final class ParserFactory {
                 return new YqlParser(environment);
             case SELECT:
                 return new SelectParser(environment);
+            case WEAKAND:
+                return new AllParser(environment, true);
             default:
                 throw new UnsupportedOperationException(type.toString());
         }
diff --git a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
index 649d678db55..93cac27059e 100644
--- a/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
+++ b/container-search/src/main/java/com/yahoo/search/yql/MinimalQueryInserter.java
@@ -18,6 +18,7 @@ import com.yahoo.search.query.parser.ParserFactory;
 import com.yahoo.search.result.ErrorMessage;
 import com.yahoo.search.searchchain.Execution;
 import com.yahoo.search.searchchain.PhaseNames;
+import com.yahoo.yolean.Exceptions;
 import com.yahoo.yolean.chain.After;
 import com.yahoo.yolean.chain.Before;
 import com.yahoo.yolean.chain.Provides;
@@ -93,7 +94,9 @@ public class MinimalQueryInserter extends Searcher {
             Parsable parsable = Parsable.fromQueryModel(query.getModel()).setQuery(query.properties().getString(YQL));
             newTree = parser.parse(parsable);
         } catch (RuntimeException e) {
-            return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not instantiate query from YQL", e));
+            return new Result(query, ErrorMessage.createInvalidQueryParameter("Could not create query from YQL: " +
+                                                                              Exceptions.toMessageString(e),
+                                                                              e));
         }
         if (parser.getOffset() != null) {
             int maxHits = query.properties().getInteger(MAX_HITS);
diff --git a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java
index 94332a52fed..69faed1da90 100644
--- a/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/semantics/test/SegmentSubstitutionTestCase.java
@@ -54,7 +54,7 @@ public class SegmentSubstitutionTestCase extends RuleBaseAbstractTestCase {
     }
 
     private static Item parseQuery(String query) {
-        AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE));
+        AllParser parser = new AllParser(new ParserEnvironment().setLinguistics(TestLinguistics.INSTANCE), false);
         return parser.parse(new Parsable().setQuery(query).setLanguage(Language.CHINESE_SIMPLIFIED)).getRoot();
     }
 
diff --git a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java
index ac0d676caf7..efaaaa5fca7 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/TermListTestCase.java
@@ -23,7 +23,7 @@ public class TermListTestCase {
     @Test
     public void testTermListInWeightedSet() {
         URIBuilder builder = searchUri();
-        builder.setParameter("myTerms", "{'1':1, '2':1, '3':1}");
+        builder.setParameter("myTerms", "{'1':1, '2':1, 3:1}");
         builder.setParameter("yql", "select * from sources * where weightedSet(user_id, @myTerms)");
         Query query = searchAndAssertNoErrors(builder);
         assertEquals("select * from sources * where weightedSet(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});",
@@ -33,7 +33,7 @@ public class TermListTestCase {
     @Test
     public void testTermListInWand() {
         URIBuilder builder = searchUri();
-        builder.setParameter("myTerms", "{'1':1, '2':1, '3':1}");
+        builder.setParameter("myTerms", "{'1':1, 2:1, '3':1}");
         builder.setParameter("yql", "select * from sources * where wand(user_id, @myTerms)");
         Query query = searchAndAssertNoErrors(builder);
         assertEquals("select * from sources * where wand(user_id, {\"1\": 1, \"2\": 1, \"3\": 1});",
diff --git a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java
index cc1a275af5a..c41de3a73f1 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/UserInputTestCase.java
@@ -72,7 +72,7 @@ public class UserInputTestCase {
     @Test
     public void testRawUserInput() {
         URIBuilder builder = searchUri();
-        builder.setParameter("yql", "select * from sources * where [{grammar: \"raw\"}]userInput(\"nal le\")");
+        builder.setParameter("yql", "select * from sources * where {grammar: \"raw\"}userInput(\"nal le\")");
         Query query = searchAndAssertNoErrors(builder);
         assertEquals("select * from sources * where default contains \"nal le\";", query.yqlRepresentation());
     }
@@ -81,7 +81,7 @@ public class UserInputTestCase {
     public void testSegmentedUserInput() {
         URIBuilder builder = searchUri();
         builder.setParameter("yql",
-                "select * from sources * where [{grammar: \"segment\"}]userInput(\"nal le\")");
+                "select * from sources * where {grammar: \"segment\"}userInput(\"nal le\")");
         Query query = searchAndAssertNoErrors(builder);
         assertEquals("select * from sources * where default contains ([{origin: {original: \"nal le\", offset: 0, length: 6}}]phrase(\"nal\", \"le\"));", query.yqlRepresentation());
     }
@@ -90,12 +90,50 @@ public class UserInputTestCase {
     public void testSegmentedNoiseUserInput() {
         URIBuilder builder = searchUri();
         builder.setParameter("yql",
-                "select * from sources * where [{grammar: \"segment\"}]userInput(\"^^^^^^^^\")");
+                "select * from sources * where {grammar: \"segment\"}userInput(\"^^^^^^^^\")");
         Query query = searchAndAssertNoErrors(builder);
         assertEquals("select * from sources * where default contains \"^^^^^^^^\";", query.yqlRepresentation());
     }
 
     @Test
+    public void testAnyParsedUserInput() {
+        URIBuilder builder = searchUri();
+        builder.setParameter("yql", "select * from sources * where {grammar: \"any\"}userInput('foo bar')");
+        Query query = searchAndAssertNoErrors(builder);
+        assertEquals("select * from sources * where (default contains \"foo\" OR default contains \"bar\");",
+                     query.yqlRepresentation());
+    }
+
+    @Test
+    public void testAllParsedUserInput() {
+        URIBuilder builder = searchUri();
+        builder.setParameter("yql", "select * from sources * where {grammar: \"all\"}userInput('foo bar')");
+        Query query = searchAndAssertNoErrors(builder);
+        assertEquals("select * from sources * where (default contains \"foo\" AND default contains \"bar\");",
+                     query.yqlRepresentation());
+    }
+
+    @Test
+    public void testWeakAndParsedUserInput() {
+        URIBuilder builder = searchUri();
+        builder.setParameter("yql", "select * from sources * where {grammar: \"weakAnd\"}userInput('foo bar')");
+        Query query = searchAndAssertNoErrors(builder);
+        assertEquals("select * from sources * where weakAnd(default contains \"foo\", default contains \"bar\");",
+                     query.yqlRepresentation());
+    }
+
+    @Test
+    public void testIllegalGrammar() {
+        URIBuilder builder = searchUri();
+        builder.setParameter("yql", "select * from sources * where {grammar: \"nonesuch\"}userInput('foo bar')");
+        Query query = new Query(builder.toString());
+        Result r = execution.search(query);
+        assertNotNull(r.hits().getError());
+        assertEquals("Could not create query from YQL: No query type 'nonesuch'",
+                     r.hits().getError().getDetailedMessage());
+    }
+
+    @Test
     public void testCustomDefaultIndexUserInput() {
         URIBuilder builder = searchUri();
         builder.setParameter("yql",
diff --git a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
index 636619bf1cc..15713dc1f97 100644
--- a/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
+++ b/container-search/src/test/java/com/yahoo/search/yql/YqlParserTestCase.java
@@ -399,7 +399,7 @@ public class YqlParserTestCase {
         assertFalse(root instanceof WordItem);
         assertTrue(root instanceof PhraseSegmentItem);
 
-        root = parse("select foo from bar where baz contains ([{grammar:\"raw\"}]\"yoni jo dima\")").getRoot();
+        root = parse("select foo from bar where baz contains ({grammar:\"raw\"}\"yoni jo dima\")").getRoot();
         assertEquals("baz:yoni jo dima", root.toString());
         assertTrue(root instanceof WordItem);
         assertFalse(root instanceof ExactStringItem);
@@ -410,7 +410,7 @@ public class YqlParserTestCase {
         AndItem andItem = (AndItem) root;
         assertEquals(3, andItem.getItemCount());
 
-        root = parse("select foo from bar where [{grammar:\"raw\"}]userInput(\"yoni jo dima\")").getRoot();
+        root = parse("select foo from bar where {grammar:\"raw\"}userInput(\"yoni jo dima\")").getRoot();
         assertTrue(root instanceof WordItem);
         assertTrue(root instanceof ExactStringItem);
         assertEquals("yoni jo dima", ((WordItem)root).getWord());
@@ -419,9 +419,9 @@ public class YqlParserTestCase {
     @Test
     public void testAccentDropping() {
         assertFalse(getRootWord("select foo from bar where baz contains " +
-                                "([ {accentDrop: false} ]\"colors\")").isNormalizable());
+                                "( {accentDrop: false} \"colors\")").isNormalizable());
         assertTrue(getRootWord("select foo from bar where baz contains " +
-                               "([ {accentDrop: true} ]\"colors\")").isNormalizable());
+                               "( {accentDrop: true} \"colors\")").isNormalizable());
         assertTrue(getRootWord("select foo from bar where baz contains " +
                                "\"colors\"").isNormalizable());
     }
@@ -429,9 +429,9 @@ public class YqlParserTestCase {
     @Test
     public void testCaseNormalization() {
         assertTrue(getRootWord("select foo from bar where baz contains " +
-                               "([ {normalizeCase: false} ]\"colors\")").isLowercased());
+                               "( {normalizeCase: false} \"colors\")").isLowercased());
         assertFalse(getRootWord("select foo from bar where baz contains " +
-                                "([ {normalizeCase: true} ]\"colors\")").isLowercased());
+                                "( {normalizeCase: true} \"colors\")").isLowercased());
         assertFalse(getRootWord("select foo from bar where baz contains " +
                                 "\"colors\"").isLowercased());
     }
@@ -440,10 +440,10 @@ public class YqlParserTestCase {
     public void testSegmentingRule() {
         assertEquals(SegmentingRule.PHRASE,
                      getRootWord("select foo from bar where baz contains " +
-                                 "([ {andSegmenting: false} ]\"colors\")").getSegmentingRule());
+                                 "( {andSegmenting: false} \"colors\")").getSegmentingRule());
         assertEquals(SegmentingRule.BOOLEAN_AND,
                      getRootWord("select foo from bar where baz contains " +
-                                 "([ {andSegmenting: true} ]\"colors\")").getSegmentingRule());
+                                 "( {andSegmenting: true} \"colors\")").getSegmentingRule());
         assertEquals(SegmentingRule.LANGUAGE_DEFAULT,
                      getRootWord("select foo from bar where baz contains " +
                                  "\"colors\"").getSegmentingRule());
@@ -453,10 +453,10 @@ public class YqlParserTestCase {
     public void testNfkc() {
         assertEquals("a\u030a",
                      getRootWord("select foo from bar where baz contains " +
-                                 "([ {nfkc: false} ]\"a\\u030a\")").getWord());
+                                 "( {nfkc: false} \"a\\u030a\")").getWord());
         assertEquals("\u00e5",
                      getRootWord("select foo from bar where baz contains " +
-                                 "([ {nfkc: true} ]\"a\\u030a\")").getWord());
+                                 "( {nfkc: true} \"a\\u030a\")").getWord());
         assertEquals("No NKFC by default", 
                      "a\u030a",
                      getRootWord("select foo from bar where baz contains " +
@@ -465,19 +465,19 @@ public class YqlParserTestCase {
 
     @Test
     public void testImplicitTransforms() {
-        assertFalse(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " +
-                                "false} ]\"cox\")").isFromQuery());
-        assertTrue(getRootWord("select foo from bar where baz contains ([ {implicitTransforms: " +
-                               "true} ]\"cox\")").isFromQuery());
+        assertFalse(getRootWord("select foo from bar where baz contains ({implicitTransforms: " +
+                                "false} \"cox\")").isFromQuery());
+        assertTrue(getRootWord("select foo from bar where baz contains ({implicitTransforms: " +
+                               "true} \"cox\")").isFromQuery());
         assertTrue(getRootWord("select foo from bar where baz contains \"cox\"").isFromQuery());
     }
 
     @Test
     public void testConnectivity() {
         QueryTree parsed = parse("select foo from bar where " +
-                                 "title contains ([{id: 1, connectivity: {\"id\": 3, weight: 7.0}}]\"madonna\") " +
-                                 "and title contains ([{id: 2}]\"saint\") " +
-                                 "and title contains ([{id: 3}]\"angel\")");
+                                 "title contains ({id: 1, connectivity: {\"id\": 3, weight: 7.0}}\"madonna\") " +
+                                 "and title contains ({id: 2}\"saint\") " +
+                                 "and title contains ({id: 3}\"angel\")");
         assertEquals("AND title:madonna title:saint title:angel",
                      parsed.toString());
         AndItem root = (AndItem)parsed.getRoot();
@@ -489,7 +489,7 @@ public class YqlParserTestCase {
         assertNull(second.getConnectedItem());
 
         assertParseFail("select foo from bar where " +
-                        "title contains ([{id: 1, connectivity: {id: 4, weight: 7.0}}]\"madonna\") " +
+                        "title contains ({id: 1, connectivity: {id: 4, weight: 7.0}}\"madonna\") " +
                         "and title contains ({id: 2}\"saint\") " +
                         "and title contains ({id: 3}\"angel\")",
                         new IllegalArgumentException("Item 'title:madonna' was specified to connect to item with ID 4, " +