Add grammar:none

6 files changed, 76 insertions, 5 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java
new file mode 100644
index 00000000000..70e5ba66c6a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.query.parser;
+
+import com.yahoo.prelude.query.BlockItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.query.parser.ParserEnvironment;
+
+import static com.yahoo.prelude.query.parser.Token.Kind.MINUS;
+import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER;
+import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE;
+import static com.yahoo.prelude.query.parser.Token.Kind.WORD;
+
+/**
+ * A parser which turns contiguous searchable character into tokens and filters out other characters.
+ * The resulting tokens are collected into a WeakAnd item.
+ *
+ * @author bratseth
+ */
+public final class NoGrammarParser extends AbstractParser {
+
+    public NoGrammarParser(ParserEnvironment environment) {
+        super(environment);
+    }
+
+    @Override
+    protected Item parseItems() {
+        WeakAndItem weakAnd = new WeakAndItem();
+        Token token;
+        while (null != (token = tokens.next())) {
+            Item termItem = toTerm(token);
+            if (termItem != null)
+                weakAnd.addItem(termItem);
+        }
+        return weakAnd;
+    }
+
+    /** Returns the item representing this token if it is searchable, and null otherwise */
+    private Item toTerm(Token token) {
+        if (token.kind == WORD)
+            return segment("", token, false);
+        else if (token.kind == NUMBER)
+            return new IntItem(token.image);
+        else
+            return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
index f2a0fdba7ff..6a005bc0ec9 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
@@ -13,7 +13,6 @@ import java.util.Set;
 
 /**
  * @author Simon Thoresen Hult
- * @since 5.1.4
  */
 public final class ProgrammaticParser implements CustomParser {
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
index 498b3185489..a35ad91acbd 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
@@ -7,7 +7,6 @@ import com.yahoo.search.query.parser.ParserEnvironment;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import static com.yahoo.prelude.query.parser.Token.Kind.*;
 
@@ -444,7 +443,7 @@ abstract class StructuredParser extends AbstractParser {
      *
      * @param quoted whether this token is inside quotes
      */
-    private Item word(String indexName, boolean quoted) {
+    protected Item word(String indexName, boolean quoted) {
         int position = tokens.getPosition();
         Item item = null;
 
@@ -470,7 +469,7 @@ abstract class StructuredParser extends AbstractParser {
                     if (buffer == null) {
                         buffer = getStringContents(item);
                     }
-                    buffer.append(token.toString());
+                    buffer.append(token);
                     tokens.skipNoIgnore();
                     token = tokens.currentNoIgnore();
                 }
diff --git a/container-search/src/main/java/com/yahoo/search/Query.java b/container-search/src/main/java/com/yahoo/search/Query.java
index 1c38bb8f876..83be5e7f6cd 100644
--- a/container-search/src/main/java/com/yahoo/search/Query.java
+++ b/container-search/src/main/java/com/yahoo/search/Query.java
@@ -109,7 +109,8 @@ public class Query extends com.yahoo.processing.Request implements Cloneable {
         PROGRAMMATIC(5, "prog"),
         YQL(6, "yql"),
         SELECT(7, "select"),
-        WEAKAND(8, "weakAnd");
+        WEAKAND(8, "weakAnd"),
+        NONE(9, "none");
 
         private final int intValue;
         private final String stringValue;
diff --git a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
index f9b8f1785db..4d1cdc320cc 100644
--- a/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
+++ b/container-search/src/main/java/com/yahoo/search/query/parser/ParserFactory.java
@@ -46,6 +46,8 @@ public final class ParserFactory {
                 return new SelectParser(environment);
             case WEAKAND:
                 return new AllParser(environment, true);
+            case NONE:
+                return new NoGrammarParser(environment);
             default:
                 throw new UnsupportedOperationException(type.toString());
         }
diff --git a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
index 18a73625967..7f51cfa1667 100644
--- a/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
+++ b/container-search/src/test/java/com/yahoo/prelude/query/parser/test/ParseTestCase.java
@@ -2563,4 +2563,23 @@ public class ParseTestCase {
         tester.assertParsed("AND title:199 title:119 title:201 title:149", "title:１９９．１１９．２０１．１４９", Query.Type.ALL);
     }
 
+    @Test
+    public void testNoGrammar1() {
+        tester.assertParsed("WEAKAND(100) foobar", "foobar", Query.Type.NONE);
+    }
+
+    @Test
+    public void testNoGrammar2() {
+        tester.assertParsed("WEAKAND(100) foobar", "-foobar", Query.Type.NONE);
+    }
+
+    @Test
+    public void testNoGrammar3() {
+        tester.assertParsed("WEAKAND(100) foo bar", "foo -bar", Query.Type.NONE);
+    }
+
+    @Test
+    public void testNoGrammar4() {
+        tester.assertParsed("WEAKAND(100) foo bar baz one two 37", "foo -(bar baz \"one two\" 37)", Query.Type.NONE);
+    }
 }