Add grammar:none

3 files changed, 53 insertions, 4 deletions

diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java
new file mode 100644
index 00000000000..70e5ba66c6a
--- /dev/null
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/NoGrammarParser.java
@@ -0,0 +1,51 @@
+// Copyright Yahoo. Licensed under the terms of the Apache 2.0 license. See LICENSE in the project root.
+package com.yahoo.prelude.query.parser;
+
+import com.yahoo.prelude.query.BlockItem;
+import com.yahoo.prelude.query.IntItem;
+import com.yahoo.prelude.query.Item;
+import com.yahoo.prelude.query.Substring;
+import com.yahoo.prelude.query.WeakAndItem;
+import com.yahoo.prelude.query.WordItem;
+import com.yahoo.search.query.parser.ParserEnvironment;
+
+import static com.yahoo.prelude.query.parser.Token.Kind.MINUS;
+import static com.yahoo.prelude.query.parser.Token.Kind.NUMBER;
+import static com.yahoo.prelude.query.parser.Token.Kind.UNDERSCORE;
+import static com.yahoo.prelude.query.parser.Token.Kind.WORD;
+
+/**
+ * A parser which turns contiguous searchable character into tokens and filters out other characters.
+ * The resulting tokens are collected into a WeakAnd item.
+ *
+ * @author bratseth
+ */
+public final class NoGrammarParser extends AbstractParser {
+
+    public NoGrammarParser(ParserEnvironment environment) {
+        super(environment);
+    }
+
+    @Override
+    protected Item parseItems() {
+        WeakAndItem weakAnd = new WeakAndItem();
+        Token token;
+        while (null != (token = tokens.next())) {
+            Item termItem = toTerm(token);
+            if (termItem != null)
+                weakAnd.addItem(termItem);
+        }
+        return weakAnd;
+    }
+
+    /** Returns the item representing this token if it is searchable, and null otherwise */
+    private Item toTerm(Token token) {
+        if (token.kind == WORD)
+            return segment("", token, false);
+        else if (token.kind == NUMBER)
+            return new IntItem(token.image);
+        else
+            return null;
+    }
+
+}
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
index f2a0fdba7ff..6a005bc0ec9 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/ProgrammaticParser.java
@@ -13,7 +13,6 @@ import java.util.Set;
 
 /**
  * @author Simon Thoresen Hult
- * @since 5.1.4
  */
 public final class ProgrammaticParser implements CustomParser {
 
diff --git a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
index 498b3185489..a35ad91acbd 100644
--- a/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
+++ b/container-search/src/main/java/com/yahoo/prelude/query/parser/StructuredParser.java
@@ -7,7 +7,6 @@ import com.yahoo.search.query.parser.ParserEnvironment;
 
 import java.util.ArrayList;
 import java.util.List;
-import java.util.Set;
 
 import static com.yahoo.prelude.query.parser.Token.Kind.*;
 
@@ -444,7 +443,7 @@ abstract class StructuredParser extends AbstractParser {
      *
      * @param quoted whether this token is inside quotes
      */
-    private Item word(String indexName, boolean quoted) {
+    protected Item word(String indexName, boolean quoted) {
         int position = tokens.getPosition();
         Item item = null;
 
@@ -470,7 +469,7 @@ abstract class StructuredParser extends AbstractParser {
                     if (buffer == null) {
                         buffer = getStringContents(item);
                     }
-                    buffer.append(token.toString());
+                    buffer.append(token);
                     tokens.skipNoIgnore();
                     token = tokens.currentNoIgnore();
                 }